





import argparse
import json
import os

import tensorstore as ts
import torch
from flax import serialization
from flax.traverse_util import flatten_dict, unflatten_dict
from tensorflow.io import gfile

from transformers.modeling_utils import dtype_byte_size
from transformers.models.switch_transformers.convert_switch_transformers_original_flax_checkpoint_to_pytorch import (
    rename_keys,
)
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
from transformers.utils.hub import convert_file_size_to_int






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 3:
          # expert layer
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							flax_key_tuple[:-1] + ("weight",)
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							torch.permute(lowerCamelCase_,       (0, 2, 1)	)
     elif flax_key_tuple[-1] == "kernel" and ".".join(lowerCamelCase_	):
          # linear layer
          _SCREAMING_SNAKE_CASE       :	Dict						    =							flax_key_tuple[:-1] + ("weight",)
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							flax_tensor.T
     elif flax_key_tuple[-1] in ["scale", "embedding"]:
          _SCREAMING_SNAKE_CASE       :	Any						    =							flax_key_tuple[:-1] + ("weight",)

     return flax_key_tuple, flax_tensor






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	):
     if "metadata" in layer:
          _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							layer.split("metadata"	)
          _SCREAMING_SNAKE_CASE       :	int						    =							"".join(split_layer[0]	)[:-1]
          _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							[tuple(("metadata" + split_layer[1]).split("/"	)	)]
     elif "kvstore" in layer:
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							layer.split("kvstore"	)
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							"".join(split_layer[0]	)[:-1]
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							[tuple(("kvstore" + split_layer[1]).split("/"	)	)]

     else:
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							layer.split("/"	)
          _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"/".join(split_layer[:-1]	)
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							(split_layer[-1],)

     if "kvstore/path" in layer:
          _SCREAMING_SNAKE_CASE       :	Dict						    =							f"""{switch_checkpoint_path}/{checkpoint_info[layer]}"""
     elif "kvstore/driver" in layer:
          _SCREAMING_SNAKE_CASE       :	Any						    =							"file"
     else:
          _SCREAMING_SNAKE_CASE       :	str						    =							checkpoint_info[layer]

     return curr_real_layer_name, split_layer, content






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							rename_keys(lowerCamelCase_	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							{}
     for k, v in current_block.items():
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							v
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							new_current_block
     torch.save(lowerCamelCase_,       lowerCamelCase_	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase = WEIGHTS_NAME	):
     _SCREAMING_SNAKE_CASE       :	Any						    =							convert_file_size_to_int(lowerCamelCase_	)
     _SCREAMING_SNAKE_CASE       :	int						    =							[]
     _SCREAMING_SNAKE_CASE       :	Dict						    =							{}
     _SCREAMING_SNAKE_CASE       :	int						    =							0
     _SCREAMING_SNAKE_CASE       :	Dict						    =							0

     os.makedirs(lowerCamelCase_,       exist_ok=lowerCamelCase_	)
     with gfile.GFile(switch_checkpoint_path + "/checkpoint",       "rb"	) as fp:
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							serialization.msgpack_restore(fp.read()	)["optimizer"]["target"]
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							flatten_dict(lowerCamelCase_,       sep="/"	)

     _SCREAMING_SNAKE_CASE       :	Any						    =							{}
     for layer in checkpoint_info.keys():
          _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	List[Any]						    =							get_key_and_tensorstore_dict(
              lowerCamelCase_,       lowerCamelCase_,       lowerCamelCase_	)
          if curr_real_layer_name in all_layers:
               _SCREAMING_SNAKE_CASE       :	str						    =							content
          else:
               _SCREAMING_SNAKE_CASE       :	int						    =							{split_layer[-1]: content}

     for key in all_layers.keys():
          # open tensorstore file
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							ts.open(unflatten_dict(all_layers[key]	)	).result().read().result()
          _SCREAMING_SNAKE_CASE       :	Any						    =							torch.tensor(lowerCamelCase_	)
          _SCREAMING_SNAKE_CASE       :	Dict						    =							raw_weights.numel() * dtype_byte_size(raw_weights.dtype	)

          # use the renaming pattern from the small conversion scripts
          _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Any						    =							rename_base_flax_keys(tuple(key.split("/"	)	),       lowerCamelCase_	)
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							"/".join(lowerCamelCase_	)

          # If this weight is going to tip up over the maximal size, we split.
          if current_block_size + weight_size > max_shard_size:
               _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(
                   lowerCamelCase_,       weights_name.replace(".bin",       f"""-{len(lowerCamelCase_	)+1:05d}-of-???.bin"""	)	)
               rename_and_save_block(lowerCamelCase_,       lowerCamelCase_	)
               sharded_state_dicts.append(current_block.keys()	)
               del current_block
               _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							{}
               _SCREAMING_SNAKE_CASE       :	List[Any]						    =							0

          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							raw_weights.to(getattr(lowerCamelCase_,       lowerCamelCase_	)	)
          current_block_size += weight_size
          total_size += weight_size

     # Add the last block
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							os.path.join(lowerCamelCase_,       weights_name.replace(".bin",       f"""-{len(lowerCamelCase_	)+1:05d}-of-???.bin"""	)	)
     rename_and_save_block(lowerCamelCase_,       lowerCamelCase_	)
     sharded_state_dicts.append(current_block.keys()	)

     # If we only have one shard, we return it
     if len(lowerCamelCase_	) == 1:
          return {weights_name: sharded_state_dicts[0]}, None

     # Otherwise, let's build the index
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							{}
     _SCREAMING_SNAKE_CASE       :	str						    =							{}
     for idx, shard in enumerate(lowerCamelCase_	):
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							weights_name.replace(
              ".bin",       f"""-{idx+1:05d}-of-{len(lowerCamelCase_	):05d}.bin"""	)  # len(sharded_state_dicts):05d}
          _SCREAMING_SNAKE_CASE       :	str						    =							os.path.join(lowerCamelCase_,       weights_name.replace(".bin",       f"""-{idx+1:05d}-of-???.bin"""	)	)
          os.rename(lowerCamelCase_,       os.path.join(lowerCamelCase_,       lowerCamelCase_	)	)
          _SCREAMING_SNAKE_CASE       :	Dict						    =							shard
          for key in shard:
               _SCREAMING_SNAKE_CASE       :	Any						    =							shard_file

    # Add the metadata
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							{"total_size": total_size}
     _SCREAMING_SNAKE_CASE       :	Any						    =							{"metadata": metadata, "weight_map": weight_map}

     with open(os.path.join(lowerCamelCase_,       lowerCamelCase_	),       "w",       encoding="utf-8"	) as f:
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							json.dumps(lowerCamelCase_,       indent=2,       sort_keys=lowerCamelCase_	) + "\n"
          f.write(lowerCamelCase_	)

     return metadata, index


if __name__ == "__main__":
       UpperCamelCase__      =argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '--switch_t5x_checkpoint_path',
           default='/mnt/disks/disk_switch/original_checkpoints/switch-xxl-128/checkpoint_634600',
           type=str,
           required=False,
           help='Path to a directory containing a folder per layer. Follows the original Google format.',
       )
       parser.add_argument('--max_shard_size', default='10GB', required=False, help='Max shard size')
       parser.add_argument('--dtype', default='bfloat16', type=str, required=False, help='dtype of the saved model')
       parser.add_argument(
           '--pytorch_dump_folder_path',
           default='/mnt/disks/disk_switch/original_checkpoints/switch-xxl-128-converted',
           type=str,
           required=False,
           help='Path to the output pytorch model.',
       )
       UpperCamelCase__      =parser.parse_args()
       shard_on_the_fly(
           args.switch_tax_checkpoint_path,
           args.pytorch_dump_folder_path,
           args.max_shard_size,
           args.dtype,
       )






def     lowerCamelCase__			():
     from transformers import SwitchTransformersConfig, SwitchTransformersForConditionalGeneration, TaTokenizer

     _SCREAMING_SNAKE_CASE       :	Any						    =							SwitchTransformersConfig.from_pretrained("google/switch-base-8"	)
     config.save_pretrained("/home/arthur_huggingface_co/transformers/switch_converted"	)
     _SCREAMING_SNAKE_CASE       :	int						    =							SwitchTransformersForConditionalGeneration.from_pretrained(
         "/home/arthur_huggingface_co/transformers/switch_converted",       device_map="auto"	)

     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							TaTokenizer.from_pretrained("t5-small"	)
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							"A <extra_id_0> walks into a bar a orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>."

     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenizer(lowerCamelCase_,       return_tensors="pt"	).input_ids
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							model.generate(lowerCamelCase_,       decoder_start_token_id=0	)
     print(tokenizer.decode(out[0]	)	)











from __future__ import annotations

import requests






def     lowerCamelCase__			(__lowerCamelCase	):
     _SCREAMING_SNAKE_CASE       :	Dict						    =							f"""https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty"""
     return requests.get(__lowerCamelCase	).json()






def     lowerCamelCase__			(__lowerCamelCase = 10	):
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							"https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty"
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							requests.get(__lowerCamelCase	).json()[:max_stories]
     return [get_hackernews_story(__lowerCamelCase	) for story_id in story_ids]






def     lowerCamelCase__			(__lowerCamelCase = 10	):
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							hackernews_top_stories(__lowerCamelCase	)
     return "\n".join("* [{title}]({url})".format(**__lowerCamelCase	) for story in stories	)


if __name__ == "__main__":
       print(hackernews_top_stories_as_markdown())