





"""simple docstring"""



import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse("""0.8.3"""):
       raise Exception("""requires gluonnlp == 0.8.3""")

if version.parse(mx.__version__) != version.parse("""1.5.0"""):
       raise Exception("""requires mxnet == 1.5.0""")

logging.set_verbosity_info()
a_			=      logging.get_logger(__name__)

a_			=      """The Nymphenburg Palace is a beautiful palace in Munich!"""







def 		__lowercase					(       snake_case_  :  str    ,snake_case_  :  str						)       ->Tuple:



    '''simple docstring'''







    __A   :						Optional[int]									=					{
        '''attention_cell''': '''multi_head''',
        '''num_layers''': 4,
        '''units''': 1024,
        '''hidden_size''': 768,
        '''max_length''': 512,
        '''num_heads''': 8,
        '''scaled''': True,
        '''dropout''': 0.1,
        '''use_residual''': True,
        '''embed_size''': 1024,
        '''embed_dropout''': 0.1,
        '''word_embed''': None,
        '''layer_norm_eps''': 1e-5,
        '''token_type_vocab_size''': 2,
    }

    __A   :						Tuple									=					bort_4_8_768_1024_hparams

    # Let's construct the original Bort model here
    # Taken from official BERT implementation, see:
    # https://github.com/alexa/bort/blob/master/bort/bort.py
    __A   :						str									=					BERTEncoder(
        attention_cell=predefined_args['''attention_cell''']    ,num_layers=predefined_args['''num_layers''']    ,units=predefined_args['''units''']    ,hidden_size=predefined_args['''hidden_size''']    ,max_length=predefined_args['''max_length''']    ,num_heads=predefined_args['''num_heads''']    ,scaled=predefined_args['''scaled''']    ,dropout=predefined_args['''dropout''']    ,output_attention=snake_case_    ,output_all_encodings=snake_case_    ,use_residual=predefined_args['''use_residual''']    ,activation=predefined_args.get('''activation'''    ,'''gelu'''						)    ,layer_norm_eps=predefined_args.get('''layer_norm_eps'''    ,snake_case_						)    ,)

    # Vocab information needs to be fetched first
    # It's the same as RoBERTa, so RobertaTokenizer can be used later
    __A   :						Tuple									=					'''openwebtext_ccnews_stories_books_cased'''

    # Specify download folder to Gluonnlp's vocab
    __A   :						Tuple									=					os.path.join(get_home_dir()    ,'''models'''						)
    __A   :						Optional[Any]									=					_load_vocab(snake_case_    ,snake_case_    ,snake_case_    ,cls=snake_case_						)

    __A   :						Tuple									=					nlp.model.BERTModel(
        snake_case_    ,len(snake_case_						)    ,units=predefined_args['''units''']    ,embed_size=predefined_args['''embed_size''']    ,embed_dropout=predefined_args['''embed_dropout''']    ,word_embed=predefined_args['''word_embed''']    ,use_pooler=snake_case_    ,use_token_type_embed=snake_case_    ,token_type_vocab_size=predefined_args['''token_type_vocab_size''']    ,use_classifier=snake_case_    ,use_decoder=snake_case_    ,)

    original_bort.load_parameters(snake_case_    ,cast_dtype=snake_case_    ,ignore_extra=snake_case_						)
    __A   :						int									=					original_bort._collect_params_with_prefix()

    # Build our config 🤗
    __A   :						Tuple									=					{
        '''architectures''': ['''BertForMaskedLM'''],
        '''attention_probs_dropout_prob''': predefined_args['''dropout'''],
        '''hidden_act''': '''gelu''',
        '''hidden_dropout_prob''': predefined_args['''dropout'''],
        '''hidden_size''': predefined_args['''embed_size'''],
        '''initializer_range''': 0.02,
        '''intermediate_size''': predefined_args['''hidden_size'''],
        '''layer_norm_eps''': predefined_args['''layer_norm_eps'''],
        '''max_position_embeddings''': predefined_args['''max_length'''],
        '''model_type''': '''bort''',
        '''num_attention_heads''': predefined_args['''num_heads'''],
        '''num_hidden_layers''': predefined_args['''num_layers'''],
        '''pad_token_id''': 1,  # 2 = BERT, 1 = RoBERTa
        '''type_vocab_size''': 1,  # 2 = BERT, 1 = RoBERTa
        '''vocab_size''': len(snake_case_						),
    }

    __A   :						Dict									=					BertConfig.from_dict(snake_case_						)
    __A   :						str									=					BertForMaskedLM(snake_case_						)
    hf_bort_model.eval()

    # Parameter mapping table (Gluonnlp to Transformers)
    # * denotes layer index
    #
    # | Gluon Parameter                                                | Transformers Parameter
    # | -------------------------------------------------------------- | ----------------------
    # | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
    # | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
    # | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
    # | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
    # | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
    # | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
    # | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
    # | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
    # | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
    # | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
    # | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
    # | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
    # | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
    # | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
    # | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
    # | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
    # | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
    # | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
    # | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
    # | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

    # Helper function to convert MXNET Arrays to PyTorch
    def to_torch(snake_case_  :  Dict						) -> nn.Parameter:
        return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()						)						)

    # Check param shapes and map new HF param back
    def check_and_map_params(snake_case_  :  Dict    ,snake_case_  :  Tuple						):
        __A   :						Union[str, Any]									=					hf_param.shape

        __A   :						List[str]									=					to_torch(params[gluon_param]						)
        __A   :						int									=					gluon_param.shape

        assert (
            shape_hf == shape_gluon
        ), F"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

        return gluon_param

    __A   :						Tuple									=					check_and_map_params(
        hf_bort_model.bert.embeddings.word_embeddings.weight    ,'''word_embed.0.weight'''						)
    __A   :						Tuple									=					check_and_map_params(
        hf_bort_model.bert.embeddings.position_embeddings.weight    ,'''encoder.position_weight'''						)
    __A   :						Dict									=					check_and_map_params(
        hf_bort_model.bert.embeddings.LayerNorm.bias    ,'''encoder.layer_norm.beta'''						)
    __A   :						Optional[int]									=					check_and_map_params(
        hf_bort_model.bert.embeddings.LayerNorm.weight    ,'''encoder.layer_norm.gamma'''						)

    # Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
    __A   :						Dict									=					torch.zeros_like(
        hf_bort_model.bert.embeddings.token_type_embeddings.weight.data						)

    for i in range(hf_bort_config.num_hidden_layers						):
        __A   :						BertLayer									=					hf_bort_model.bert.encoder.layer[i]

        # self attention
        __A   :						BertSelfAttention									=					layer.attention.self

        __A   :						Optional[int]									=					check_and_map_params(
            self_attn.key.bias.data    ,F"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""						)

        __A   :						Union[str, Any]									=					check_and_map_params(
            self_attn.key.weight.data    ,F"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""						)
        __A   :						int									=					check_and_map_params(
            self_attn.query.bias.data    ,F"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""						)
        __A   :						Union[str, Any]									=					check_and_map_params(
            self_attn.query.weight.data    ,F"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""						)
        __A   :						Dict									=					check_and_map_params(
            self_attn.value.bias.data    ,F"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""						)
        __A   :						Union[str, Any]									=					check_and_map_params(
            self_attn.value.weight.data    ,F"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""						)

        # self attention output
        __A   :						BertSelfOutput									=					layer.attention.output

        __A   :						List[Any]									=					check_and_map_params(
            self_output.dense.bias    ,F"""encoder.transformer_cells.{i}.proj.bias"""						)
        __A   :						Optional[Any]									=					check_and_map_params(
            self_output.dense.weight    ,F"""encoder.transformer_cells.{i}.proj.weight"""						)
        __A   :						str									=					check_and_map_params(
            self_output.LayerNorm.bias    ,F"""encoder.transformer_cells.{i}.layer_norm.beta"""						)
        __A   :						List[Any]									=					check_and_map_params(
            self_output.LayerNorm.weight    ,F"""encoder.transformer_cells.{i}.layer_norm.gamma"""						)

        # intermediate
        __A   :						BertIntermediate									=					layer.intermediate

        __A   :						List[Any]									=					check_and_map_params(
            intermediate.dense.bias    ,F"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""						)
        __A   :						Optional[int]									=					check_and_map_params(
            intermediate.dense.weight    ,F"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""						)

        # output
        __A   :						BertOutput									=					layer.output

        __A   :						Union[str, Any]									=					check_and_map_params(
            bert_output.dense.bias    ,F"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""						)
        __A   :						Tuple									=					check_and_map_params(
            bert_output.dense.weight    ,F"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""						)
        __A   :						List[Any]									=					check_and_map_params(
            bert_output.LayerNorm.bias    ,F"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""						)
        __A   :						Optional[Any]									=					check_and_map_params(
            bert_output.LayerNorm.weight    ,F"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""						)

    # Save space and energy 🎄
    hf_bort_model.half()

    # Compare output of both models
    __A   :						Dict									=					RobertaTokenizer.from_pretrained('''roberta-base'''						)

    __A   :						Any									=					tokenizer.encode_plus(snake_case_						)['''input_ids''']

    # Get gluon output
    __A   :						Dict									=					mx.nd.array([input_ids]						)
    __A   :						Optional[int]									=					original_bort(inputs=snake_case_    ,token_types=[]						)

    # Get Transformer output (save and reload model again)
    hf_bort_model.save_pretrained(snake_case_						)
    __A   :						Optional[Any]									=					BertModel.from_pretrained(snake_case_						)
    hf_bort_model.eval()

    __A   :						Optional[int]									=					tokenizer.encode_plus(snake_case_    ,return_tensors='''pt'''						)
    __A   :						Dict									=					hf_bort_model(**snake_case_						)[0]

    __A   :						Dict									=					output_gluon[0].asnumpy()
    __A   :						int									=					output_hf[0].detach().numpy()

    __A   :						List[Any]									=					np.max(np.abs(hf_layer - gluon_layer						)						).item()
    __A   :						Tuple									=					np.allclose(snake_case_    ,snake_case_    ,atol=1e-3						)

    if success:
        print('''✔️ Both model do output the same tensors'''						)
    else:
        print('''❌ Both model do **NOT** output the same tensors'''						)
        print('''Absolute difference is:'''    ,snake_case_						)


if __name__ == "__main__":
       a_			=      argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           """--bort_checkpoint_path""", default=None, type=str, required=True, help="""Path the official Bort params file."""
       )
       parser.add_argument(
           """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
       )
       a_			=      parser.parse_args()
       convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)











"""simple docstring"""



import math
from collections.abc import Iterator
from itertools import takewhile







def 		__lowercase					(       snake_case_  :  int						)       ->bool:



    '''simple docstring'''







    if 1 < number < 4:
        # 2 and 3 are primes
        return True
    elif number < 2 or number % 2 == 0 or number % 3 == 0:
        # Negatives, 0, 1, all even numbers, all multiples of 3 are not primes
        return False

    # All primes number are in format of 6k +/- 1
    for i in range(5    ,int(math.sqrt(snake_case_						) + 1						)    ,6						):
        if number % i == 0 or number % (i + 2) == 0:
            return False
    return True







def 		__lowercase					(       )       ->Iterator[int]:



    '''simple docstring'''







    __A   :						int									=					2
    while True:
        if is_prime(snake_case_						):
            yield num
        num += 1







def 		__lowercase					(       snake_case_  :  int = 2000000						)       ->int:



    '''simple docstring'''







    return sum(takewhile(lambda snake_case_						: x < n    ,prime_generator()						)						)


if __name__ == "__main__":
       print(f'''{solution() = }''')




