model_div.py
def configure_optimizers(self, train_config):
        """
        This long function is unfortunately doing something very simple and is being very defensive:
        We are separating out all parameters of the model into two buckets: those that will experience
        weight decay for regularization and those that won't (biases, and layernorm/embedding weights).
        We are then returning the PyTorch optimizer object.
        """

        # separate out all parameters to those that will and won't experience regularizing weight decay
        decay = set()
        no_decay = set()
        whitelist_weight_modules = (torch.nn.Linear, torch.nn.LSTM)
        blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding)
        for mn, m in self.named_modules():
            for pn, p in m.named_parameters():
                fpn = '%s.%s' % (mn, pn) if mn else pn # full param name

                if pn.endswith('bias') or ('bias' in pn):
                    # all biases will not be decayed
                    no_decay.add(fpn)
                elif (pn.endswith('weight') or ('weight' in pn)) and isinstance(m, whitelist_weight_modules):
                    # weights of whitelist modules will be weight decayed
                    decay.add(fpn)
                elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules):
                    # weights of blacklist modules will NOT be weight decayed
                    no_decay.add(fpn)

        # special case the position embedding parameter in the root GPT module as not decayed
        no_decay.add('pos_emb')
        
        # validate that we considered every parameter
        param_dict = {pn: p for pn, p in self.named_parameters()}
        inter_params = decay & no_decay
        union_params = decay | no_decay
        assert len(inter_params) == 0, "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), )
        assert len(param_dict.keys() - union_params) == 0, "parameters %s were not separated into either decay/no_decay set!" \
                                                    % (str(param_dict.keys() - union_params), )

        # create the pytorch optimizer object
        optim_groups = [
            {"params": [param_dict[pn] for pn in sorted(list(decay))], "weight_decay": train_config.weight_decay},
            {"params": [param_dict[pn] for pn in sorted(list(no_decay))], "weight_decay": 0.0},
        ]
        optimizer = torch.optim.AdamW(optim_groups, lr=train_config.learning_rate, betas=train_config.betas)
        return optimizer

        This long function is unfortunately doing something very simple and is being very defensive:
        We are separating out all parameters of the model into two buckets: those that will experience
        weight decay for regularization and those that won't (biases, and layernorm/embedding weights).
        We are then returning the PyTorch optimizer object.
        
This long function is unfortunately doing something very simple and is being very defensive:
We are separating out all parameters of the model into two buckets: those that will experience
weight decay for regularization and those that won't (biases, and layernorm/embedding weights).
We are then returning the PyTorch optimizer object.
This long function is unfortunately doing something very simple and is being very defensive : We are separating out all parameters of the model into two buckets : those that will experience weight decay for regularization and those that won ' t ( biases and layernorm / embedding weights ) . We are then returning the PyTorch optimizer object .
def configure_optimizers(self, train_config):
        decay = set()
        no_decay = set()
        whitelist_weight_modules = (torch.nn.Linear, torch.nn.LSTM)
        blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding)
        for mn, m in self.named_modules():
            for pn, p in m.named_parameters():
                fpn = '%s.%s' % (mn, pn) if mn else pn 
                if pn.endswith('bias') or ('bias' in pn):
                    no_decay.add(fpn)
                elif (pn.endswith('weight') or ('weight' in pn)) and isinstance(m, whitelist_weight_modules):
                    decay.add(fpn)
                elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules):
                    no_decay.add(fpn)
        no_decay.add('pos_emb')
        param_dict = {pn: p for pn, p in self.named_parameters()}
        inter_params = decay & no_decay
        union_params = decay | no_decay
        assert len(inter_params) == 0, "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), )
        assert len(param_dict.keys() - union_params) == 0, "parameters %s were not separated into either decay/no_decay set!" \
                                                    % (str(param_dict.keys() - union_params), )
        optim_groups = [
            {"params": [param_dict[pn] for pn in sorted(list(decay))], "weight_decay": train_config.weight_decay},
            {"params": [param_dict[pn] for pn in sorted(list(no_decay))], "weight_decay": 0.0},
        ]
        optimizer = torch.optim.AdamW(optim_groups, lr=train_config.learning_rate, betas=train_config.betas)
        return optimizer
def configure_optimizers ( self , train_config ) : decay = set ( ) no_decay = set ( ) whitelist_weight_modules = ( torch . nn . Linear , torch . nn . LSTM ) blacklist_weight_modules = ( torch . nn . LayerNorm , torch . nn . Embedding ) for mn , m in self . named_modules ( ) : for pn , p in m . named_parameters ( ) : fpn = '%s.%s' % ( mn , pn ) if mn else pn if pn . endswith ( 'bias' ) or ( 'bias' in pn ) : no_decay . add ( fpn ) elif ( pn . endswith ( 'weight' ) or ( 'weight' in pn ) ) and isinstance ( m , whitelist_weight_modules ) : decay . add ( fpn ) elif pn . endswith ( 'weight' ) and isinstance ( m , blacklist_weight_modules ) : no_decay . add ( fpn ) no_decay . add ( 'pos_emb' ) param_dict = { pn : p for pn , p in self . named_parameters ( ) } inter_params = decay & no_decay union_params = decay | no_decay assert len ( inter_params ) == 0 , "parameters %s made it into both decay/no_decay sets!" % ( str ( inter_params ) , ) assert len ( param_dict . keys ( ) - union_params ) == 0 , "parameters %s were not separated into either decay/no_decay set!" % ( str ( param_dict . keys ( ) - union_params ) , ) optim_groups = [ { "params" : [ param_dict [ pn ] for pn in sorted ( list ( decay ) ) ] , "weight_decay" : train_config . weight_decay } , { "params" : [ param_dict [ pn ] for pn in sorted ( list ( no_decay ) ) ] , "weight_decay" : 0.0 } , ] optimizer = torch . optim . AdamW ( optim_groups , lr = train_config . learning_rate , betas = train_config . betas ) return optimizer
This long function is unfortunately doing something very simple and is being very defensive:
We are separating out all parameters of the model into two buckets: those that will experience
weight decay for regularization and those that won't (biases, and layernorm/embedding weights).
This long function is unfortunately doing something very simple and is being very defensive : We are separating out all parameters of the model into two buckets : those that will experience weight decay for regularization and those that won ' t ( biases and layernorm / embedding weights ) .
"""
        This long function is unfortunately doing something very simple and is being very defensive:
        We are separating out all parameters of the model into two buckets: those that will experience
        weight decay for regularization and those that won't (biases, and layernorm/embedding weights).
        We are then returning the PyTorch optimizer object.
        """ # separate out all parameters to those that will and won't experience regularizing weight decay # full param name # all biases will not be decayed # weights of whitelist modules will be weight decayed # weights of blacklist modules will NOT be weight decayed # special case the position embedding parameter in the root GPT module as not decayed # validate that we considered every parameter # create the pytorch optimizer object
self
train_config