


import argparse
from collections import OrderedDict
from pathlib import Path

import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision.transforms import functional as F

from transformers import DetrImageProcessor, TableTransformerConfig, TableTransformerForObjectDetection
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE :   str							      =				logging.get_logger(__name__)

# here we list all keys to be renamed (original name on the left, our name on the right)
SCREAMING_SNAKE_CASE :   Optional[Any]							      =				[]
for i in range(6):
 # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
 rename_keys.append(
     (F"transformer.encoder.layers.{i}.self_attn.out_proj.weight", F"encoder.layers.{i}.self_attn.out_proj.weight")
 )
 rename_keys.append(
     (F"transformer.encoder.layers.{i}.self_attn.out_proj.bias", F"encoder.layers.{i}.self_attn.out_proj.bias")
 )
 rename_keys.append((F"transformer.encoder.layers.{i}.linear1.weight", F"encoder.layers.{i}.fc1.weight"))
 rename_keys.append((F"transformer.encoder.layers.{i}.linear1.bias", F"encoder.layers.{i}.fc1.bias"))
 rename_keys.append((F"transformer.encoder.layers.{i}.linear2.weight", F"encoder.layers.{i}.fc2.weight"))
 rename_keys.append((F"transformer.encoder.layers.{i}.linear2.bias", F"encoder.layers.{i}.fc2.bias"))
 rename_keys.append(
     (F"transformer.encoder.layers.{i}.norm1.weight", F"encoder.layers.{i}.self_attn_layer_norm.weight")
 )
 rename_keys.append((F"transformer.encoder.layers.{i}.norm1.bias", F"encoder.layers.{i}.self_attn_layer_norm.bias"))
 rename_keys.append((F"transformer.encoder.layers.{i}.norm2.weight", F"encoder.layers.{i}.final_layer_norm.weight"))
 rename_keys.append((F"transformer.encoder.layers.{i}.norm2.bias", F"encoder.layers.{i}.final_layer_norm.bias"))
 # decoder layers: 2 times output projection, 2 feedforward neural networks and 3 layernorms
 rename_keys.append(
     (F"transformer.decoder.layers.{i}.self_attn.out_proj.weight", F"decoder.layers.{i}.self_attn.out_proj.weight")
 )
 rename_keys.append(
     (F"transformer.decoder.layers.{i}.self_attn.out_proj.bias", F"decoder.layers.{i}.self_attn.out_proj.bias")
 )
 rename_keys.append(
     (
         F"transformer.decoder.layers.{i}.multihead_attn.out_proj.weight",
         F"decoder.layers.{i}.encoder_attn.out_proj.weight",
     )
 )
 rename_keys.append(
     (
         F"transformer.decoder.layers.{i}.multihead_attn.out_proj.bias",
         F"decoder.layers.{i}.encoder_attn.out_proj.bias",
     )
 )
 rename_keys.append((F"transformer.decoder.layers.{i}.linear1.weight", F"decoder.layers.{i}.fc1.weight"))
 rename_keys.append((F"transformer.decoder.layers.{i}.linear1.bias", F"decoder.layers.{i}.fc1.bias"))
 rename_keys.append((F"transformer.decoder.layers.{i}.linear2.weight", F"decoder.layers.{i}.fc2.weight"))
 rename_keys.append((F"transformer.decoder.layers.{i}.linear2.bias", F"decoder.layers.{i}.fc2.bias"))
 rename_keys.append(
     (F"transformer.decoder.layers.{i}.norm1.weight", F"decoder.layers.{i}.self_attn_layer_norm.weight")
 )
 rename_keys.append((F"transformer.decoder.layers.{i}.norm1.bias", F"decoder.layers.{i}.self_attn_layer_norm.bias"))
 rename_keys.append(
     (F"transformer.decoder.layers.{i}.norm2.weight", F"decoder.layers.{i}.encoder_attn_layer_norm.weight")
 )
 rename_keys.append(
     (F"transformer.decoder.layers.{i}.norm2.bias", F"decoder.layers.{i}.encoder_attn_layer_norm.bias")
 )
 rename_keys.append((F"transformer.decoder.layers.{i}.norm3.weight", F"decoder.layers.{i}.final_layer_norm.weight"))
 rename_keys.append((F"transformer.decoder.layers.{i}.norm3.bias", F"decoder.layers.{i}.final_layer_norm.bias"))

# convolutional projection + query embeddings + layernorm of encoder + layernorm of decoder + class and bounding box heads
rename_keys.extend(
    [
        ("input_proj.weight", "input_projection.weight"),
        ("input_proj.bias", "input_projection.bias"),
        ("query_embed.weight", "query_position_embeddings.weight"),
        ("transformer.encoder.norm.weight", "encoder.layernorm.weight"),
        ("transformer.encoder.norm.bias", "encoder.layernorm.bias"),
        ("transformer.decoder.norm.weight", "decoder.layernorm.weight"),
        ("transformer.decoder.norm.bias", "decoder.layernorm.bias"),
        ("class_embed.weight", "class_labels_classifier.weight"),
        ("class_embed.bias", "class_labels_classifier.bias"),
        ("bbox_embed.layers.0.weight", "bbox_predictor.layers.0.weight"),
        ("bbox_embed.layers.0.bias", "bbox_predictor.layers.0.bias"),
        ("bbox_embed.layers.1.weight", "bbox_predictor.layers.1.weight"),
        ("bbox_embed.layers.1.bias", "bbox_predictor.layers.1.bias"),
        ("bbox_embed.layers.2.weight", "bbox_predictor.layers.2.weight"),
        ("bbox_embed.layers.2.bias", "bbox_predictor.layers.2.bias"),
    ]
)



def        UpperCamelCase_(   lowerCamelCase_							,				lowerCamelCase_							,				lowerCamelCase_				)					->		Union[str, Any]:
    _lowercase :							List[Any]   	=	state_dict.pop(lowerCamelCase_				)
    _lowercase :							str   	=	val



def        UpperCamelCase_(   lowerCamelCase_				)					->		str:
    _lowercase :							int   	=	OrderedDict()
    for key, value in state_dict.items():
        if "backbone.0.body" in key:
            _lowercase :							Optional[int]   	=	key.replace('backbone.0.body'							,				'backbone.conv_encoder.model'				)
            _lowercase :							Dict   	=	value
        else:
            _lowercase :							List[str]   	=	value

    return new_state_dict



def        UpperCamelCase_(   lowerCamelCase_				)					->		Dict:
    _lowercase :							Any   	=	''

    # first: transformer encoder
    for i in range(6				):
        # read in weights + bias of input projection layer (in PyTorch's MultiHeadAttention, this is a single matrix + bias)
        _lowercase :							List[Any]   	=	state_dict.pop(F'''{prefix}transformer.encoder.layers.{i}.self_attn.in_proj_weight'''				)
        _lowercase :							Dict   	=	state_dict.pop(F'''{prefix}transformer.encoder.layers.{i}.self_attn.in_proj_bias'''				)
        # next, add query, keys and values (in that order) to the state dict
        _lowercase :							Optional[int]   	=	in_proj_weight[:256, :]
        _lowercase :							Any   	=	in_proj_bias[:256]
        _lowercase :							Optional[int]   	=	in_proj_weight[256:512, :]
        _lowercase :							Optional[Any]   	=	in_proj_bias[256:512]
        _lowercase :							str   	=	in_proj_weight[-256:, :]
        _lowercase :							Dict   	=	in_proj_bias[-256:]
    # next: transformer decoder (which is a bit more complex because it also includes cross-attention)
    for i in range(6				):
        # read in weights + bias of input projection layer of self-attention
        _lowercase :							Dict   	=	state_dict.pop(F'''{prefix}transformer.decoder.layers.{i}.self_attn.in_proj_weight'''				)
        _lowercase :							int   	=	state_dict.pop(F'''{prefix}transformer.decoder.layers.{i}.self_attn.in_proj_bias'''				)
        # next, add query, keys and values (in that order) to the state dict
        _lowercase :							Optional[Any]   	=	in_proj_weight[:256, :]
        _lowercase :							Tuple   	=	in_proj_bias[:256]
        _lowercase :							str   	=	in_proj_weight[256:512, :]
        _lowercase :							Optional[Any]   	=	in_proj_bias[256:512]
        _lowercase :							List[Any]   	=	in_proj_weight[-256:, :]
        _lowercase :							Optional[int]   	=	in_proj_bias[-256:]
        # read in weights + bias of input projection layer of cross-attention
        _lowercase :							Union[str, Any]   	=	state_dict.pop(
            F'''{prefix}transformer.decoder.layers.{i}.multihead_attn.in_proj_weight'''				)
        _lowercase :							Tuple   	=	state_dict.pop(F'''{prefix}transformer.decoder.layers.{i}.multihead_attn.in_proj_bias'''				)
        # next, add query, keys and values (in that order) of cross-attention to the state dict
        _lowercase :							Tuple   	=	in_proj_weight_cross_attn[:256, :]
        _lowercase :							int   	=	in_proj_bias_cross_attn[:256]
        _lowercase :							Dict   	=	in_proj_weight_cross_attn[256:512, :]
        _lowercase :							int   	=	in_proj_bias_cross_attn[256:512]
        _lowercase :							Union[str, Any]   	=	in_proj_weight_cross_attn[-256:, :]
        _lowercase :							Optional[int]   	=	in_proj_bias_cross_attn[-256:]



def        UpperCamelCase_(   lowerCamelCase_							,				lowerCamelCase_				)					->		List[str]:
    _lowercase	,       _lowercase :							List[str]   	=	image.size
    _lowercase :							Any   	=	max(lowerCamelCase_							,				lowerCamelCase_				)
    _lowercase :							Tuple   	=	800 if 'detection' in checkpoint_url else 1000
    _lowercase :							List[str]   	=	target_max_size / current_max_size
    _lowercase :							List[str]   	=	image.resize((int(round(scale * width				)				), int(round(scale * height				)				))				)

    return resized_image



def        UpperCamelCase_(   lowerCamelCase_				)					->		Any:
    _lowercase :							str   	=	F.to_tensor(lowerCamelCase_				)
    _lowercase :							Union[str, Any]   	=	F.normalize(lowerCamelCase_							,				mean=[0.4_85, 0.4_56, 0.4_06]							,				std=[0.2_29, 0.2_24, 0.2_25]				)
    return image



@torch.no_grad()
def        UpperCamelCase_(   lowerCamelCase_							,				lowerCamelCase_							,				lowerCamelCase_				)					->		str:
    logger.info('Converting model...'				)

    # load original state dict
    _lowercase :							Optional[int]   	=	torch.hub.load_state_dict_from_url(lowerCamelCase_							,				map_location='cpu'				)
    # rename keys
    for src, dest in rename_keys:
        rename_key(lowerCamelCase_							,				lowerCamelCase_							,				lowerCamelCase_				)
    _lowercase :							List[Any]   	=	rename_backbone_keys(lowerCamelCase_				)
    # query, key and value matrices need special treatment
    read_in_q_k_v(lowerCamelCase_				)
    # important: we need to prepend a prefix to each of the base model keys as the head models use different attributes for them
    _lowercase :							Optional[int]   	=	'model.'
    for key in state_dict.copy().keys():
        if not key.startswith('class_labels_classifier'				) and not key.startswith('bbox_predictor'				):
            _lowercase :							int   	=	state_dict.pop(lowerCamelCase_				)
            _lowercase :							Optional[int]   	=	val
    # create HuggingFace model and load state dict
    _lowercase :							Any   	=	TableTransformerConfig(
        backbone='resnet18'							,				mask_loss_coefficient=1							,				dice_loss_coefficient=1							,				ce_loss_coefficient=1							,				bbox_loss_coefficient=5							,				giou_loss_coefficient=2							,				eos_coefficient=0.4							,				class_cost=1							,				bbox_cost=5							,				giou_cost=2							,				)

    if "detection" in checkpoint_url:
        _lowercase :							int   	=	15
        _lowercase :							List[str]   	=	2
        _lowercase :							List[Any]   	=	{0: 'table', 1: 'table rotated'}
        _lowercase :							str   	=	idalabel
        _lowercase :							Union[str, Any]   	=	{v: k for k, v in idalabel.items()}
    else:
        _lowercase :							Dict   	=	125
        _lowercase :							Optional[int]   	=	6
        _lowercase :							Any   	=	{
            0: 'table',
            1: 'table column',
            2: 'table row',
            3: 'table column header',
            4: 'table projected row header',
            5: 'table spanning cell',
        }
        _lowercase :							List[Any]   	=	idalabel
        _lowercase :							List[Any]   	=	{v: k for k, v in idalabel.items()}

    _lowercase :							Tuple   	=	DetrImageProcessor(
        format='coco_detection'							,				max_size=800 if 'detection' in checkpoint_url else 1000				)
    _lowercase :							int   	=	TableTransformerForObjectDetection(lowerCamelCase_				)
    model.load_state_dict(lowerCamelCase_				)
    model.eval()

    # verify our conversion
    _lowercase :							Optional[Any]   	=	'example_pdf.png' if 'detection' in checkpoint_url else 'example_table.png'
    _lowercase :							Optional[int]   	=	hf_hub_download(repo_id='nielsr/example-pdf'							,				repo_type='dataset'							,				filename=lowerCamelCase_				)
    _lowercase :							Union[str, Any]   	=	Image.open(lowerCamelCase_				).convert('RGB'				)
    _lowercase :							Union[str, Any]   	=	normalize(resize(lowerCamelCase_							,				lowerCamelCase_				)				).unsqueeze(0				)

    _lowercase :							Optional[int]   	=	model(lowerCamelCase_				)

    if "detection" in checkpoint_url:
        _lowercase :							Optional[Any]   	=	(1, 15, 3)
        _lowercase :							List[Any]   	=	torch.tensor(
            [[-6.78_97, -16.99_85, 6.79_37], [-8.01_86, -22.21_92, 6.96_77], [-7.31_17, -21.07_08, 7.40_55]]				)
        _lowercase :							Optional[int]   	=	torch.tensor([[0.48_67, 0.17_67, 0.67_32], [0.67_18, 0.44_79, 0.38_30], [0.47_16, 0.17_60, 0.63_64]]				)

    else:
        _lowercase :							Dict   	=	(1, 125, 7)
        _lowercase :							Union[str, Any]   	=	torch.tensor(
            [[-18.14_30, -8.32_14, 4.82_74], [-18.46_85, -7.13_61, -4.26_67], [-26.36_93, -9.34_29, -4.99_62]]				)
        _lowercase :							List[Any]   	=	torch.tensor([[0.49_83, 0.55_95, 0.94_40], [0.49_16, 0.63_15, 0.59_54], [0.61_08, 0.86_37, 0.11_35]]				)

    assert outputs.logits.shape == expected_shape
    assert torch.allclose(outputs.logits[0, :3, :3]							,				lowerCamelCase_							,				atol=1e-4				)
    assert torch.allclose(outputs.pred_boxes[0, :3, :3]							,				lowerCamelCase_							,				atol=1e-4				)
    print('Looks ok!'				)

    if pytorch_dump_folder_path is not None:
        # Save model and image processor
        logger.info(F'''Saving PyTorch model and image processor to {pytorch_dump_folder_path}...'''				)
        Path(lowerCamelCase_				).mkdir(exist_ok=lowerCamelCase_				)
        model.save_pretrained(lowerCamelCase_				)
        image_processor.save_pretrained(lowerCamelCase_				)

    if push_to_hub:
        # Push model to HF hub
        logger.info('Pushing model to the hub...'				)
        _lowercase :							Optional[Any]   	=	(
            'microsoft/table-transformer-detection'
            if 'detection' in checkpoint_url
            else 'microsoft/table-transformer-structure-recognition'
        )
        model.push_to_hub(lowerCamelCase_				)
        image_processor.push_to_hub(lowerCamelCase_				)


if __name__ == "__main__":
 SCREAMING_SNAKE_CASE :   Tuple							      =				argparse.ArgumentParser()

 parser.add_argument(
     "--checkpoint_url",
     default="https://pubtables1m.blob.core.windows.net/model/pubtables1m_detection_detr_r18.pth",
     type=str,
     choices=[
         "https://pubtables1m.blob.core.windows.net/model/pubtables1m_detection_detr_r18.pth",
         "https://pubtables1m.blob.core.windows.net/model/pubtables1m_structure_detr_r18.pth",
     ],
     help="URL of the Table Transformer checkpoint you'd like to convert.",
 )
 parser.add_argument(
     "--pytorch_dump_folder_path", default=None, type=str, help="Path to the folder to output PyTorch model."
 )
 parser.add_argument(
     "--push_to_hub", action="store_true", help="Whether or not to push the converted model to the 🤗 hub."
 )
 SCREAMING_SNAKE_CASE :   int							      =				parser.parse_args()
 convert_table_transformer_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path, args.push_to_hub)






from __future__ import annotations

import requests



def        UpperCamelCase_(   lowerCamelCase_				)					->		dict:
    _lowercase :							Dict   	=	F'''https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty'''
    return requests.get(lowerCamelCase_				).json()



def        UpperCamelCase_(   lowerCamelCase_ = 10				)					->		list[dict]:
    _lowercase :							Dict   	=	'https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty'
    _lowercase :							Dict   	=	requests.get(lowerCamelCase_				).json()[:max_stories]
    return [get_hackernews_story(lowerCamelCase_				) for story_id in story_ids]



def        UpperCamelCase_(   lowerCamelCase_ = 10				)					->		str:
    _lowercase :							Optional[int]   	=	hackernews_top_stories(lowerCamelCase_				)
    return "\n".join('* [{title}]({url})'.format(**lowerCamelCase_				) for story in stories				)


if __name__ == "__main__":
 print(hackernews_top_stories_as_markdown())


