



"""simple docstring"""






from io import BytesIO
from typing import List, Union

import requests

from ..utils import add_end_docstrings, is_decord_available, is_torch_available, logging, requires_backends
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_decord_available():
		import numpy as np
		from decord import VideoReader


if is_torch_available():
		from ..models.auto.modeling_auto import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING

_lowercase			:			List[Any]										=						logging.get_logger(__name__)

@add_end_docstrings(lowerCAmelCase_       )
class 	__SCREAMING_SNAKE_CASE     (				lowerCAmelCase_       ):


					'''simple docstring'''





					def __init__(	self							:  Optional[Any],  *lowerCamelCase							:  int,  **lowerCamelCase							:  List[Any]					)-> Optional[int]:
								super().__init__(*lowerCamelCase,  **lowerCamelCase					)
								requires_backends(self,  '''decord'''					)
								self.check_model_type(lowerCamelCase					)



					def 		snake_case	(	self							:  Union[str, Any],  lowerCamelCase							:  Tuple=None,  lowerCamelCase							:  int=None,  lowerCamelCase							:  Any=None					)-> List[Any]:
								lowerCamelCase__		:		Any		     ={}
								if frame_sampling_rate is not None:
											lowerCamelCase__		:		Dict		     =frame_sampling_rate
								if num_frames is not None:
											lowerCamelCase__		:		Optional[int]		     =num_frames

								lowerCamelCase__		:		int		     ={}
								if top_k is not None:
											lowerCamelCase__		:		Dict		     =top_k
								return preprocess_params, {}, postprocess_params



					def __call__(	self							:  Union[str, Any],  lowerCamelCase							:  Union[str, List[str]],  **lowerCamelCase							:  str					)-> Any:
								return super().__call__(lowerCamelCase,  **lowerCamelCase					)



					def 		snake_case	(	self							:  List[str],  lowerCamelCase							:  Dict,  lowerCamelCase							:  List[str]=None,  lowerCamelCase							:  List[Any]=1					)-> Union[str, Any]:
								if num_frames is None:
											lowerCamelCase__		:		Union[str, Any]		     =self.model.config.num_frames

								if video.startswith('''http://'''					) or video.startswith('''https://'''					):
											lowerCamelCase__		:		List[str]		     =BytesIO(requests.get(lowerCamelCase					).content					)

								lowerCamelCase__		:		List[Any]		     =VideoReader(lowerCamelCase					)
								videoreader.seek(0					)

								lowerCamelCase__		:		int		     =0
								lowerCamelCase__		:		Tuple		     =num_frames * frame_sampling_rate - 1
								lowerCamelCase__		:		Optional[Any]		     =np.linspace(lowerCamelCase,  lowerCamelCase,  num=lowerCamelCase,  dtype=np.intaa					)

								lowerCamelCase__		:		List[Any]		     =videoreader.get_batch(lowerCamelCase					).asnumpy()
								lowerCamelCase__		:		Optional[Any]		     =list(lowerCamelCase					)

								lowerCamelCase__		:		str		     =self.image_processor(lowerCamelCase,  return_tensors=self.framework					)
								return model_inputs



					def 		snake_case	(	self							:  Optional[Any],  lowerCamelCase							:  Any					)-> List[str]:
								lowerCamelCase__		:		Any		     =self.model(**lowerCamelCase					)
								return model_outputs




					def 		snake_case	(	self							:  Dict,  lowerCamelCase							:  Any,  lowerCamelCase							:  List[Any]=5					)-> int:
								if top_k > self.model.config.num_labels:
											lowerCamelCase__		:		Union[str, Any]		     =self.model.config.num_labels

								if self.framework == "pt":
											lowerCamelCase__		:		int		     =model_outputs.logits.softmax(-1					)[0]
											lowerCamelCase__     ,       lowerCamelCase__		:		str		     =probs.topk(lowerCamelCase					)
								else:
											raise ValueError(F'''Unsupported framework: {self.framework}'''					)

								lowerCamelCase__		:		Tuple		     =scores.tolist()
								lowerCamelCase__		:		str		     =ids.tolist()
								return [{"score": score, "label": self.model.config.idalabel[_id]} for score, _id in zip(lowerCamelCase,  lowerCamelCase					)]









"""simple docstring"""






# Usage:
# ./gen-card-allenai-wmt16.py

import os
from pathlib import Path






def 					snake_case__       (		__lowerCamelCase    :   Union[str, Any]							,		__lowerCamelCase    :   int							,		__lowerCamelCase    :   Tuple							,		__lowerCamelCase    :   Union[str, Any]     ):
			"""simple docstring"""
			lowerCamelCase__		:		Any		     ={
			    '''en''': '''Machine learning is great, isn\'t it?''',
			    '''ru''': '''Машинное обучение - это здорово, не так ли?''',
			    '''de''': '''Maschinelles Lernen ist großartig, nicht wahr?''',
			}

			# BLUE scores as follows:
			# "pair": [fairseq, transformers]
			lowerCamelCase__		:		Optional[Any]		     ={
			    '''wmt16-en-de-dist-12-1''': [28.3, 27.52],
			    '''wmt16-en-de-dist-6-1''': [27.4, 27.11],
			    '''wmt16-en-de-12-1''': [26.9, 25.75],
			}
			lowerCamelCase__		:		Any		     =f'''{src_lang}-{tgt_lang}'''

			lowerCamelCase__		:		Any		     =f'''
---
language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt16
- allenai
license: apache-2.0
datasets:
- wmt16
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

All 3 models are available:

* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)
* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)
* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "allenai/{model_name}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   | fairseq | transformers
-------|---------|----------
{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}

The score is slightly below the score reported in the paper, as the researchers don\'t use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt16/)
- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

'''
			model_card_dir.mkdir(parents=__lowerCamelCase							,		exist_ok=__lowerCamelCase     )
			lowerCamelCase__		:		str		     =os.path.join(__lowerCamelCase							,		'''README.md'''     )
			print(f'''Generating {path}'''     )
			with open(__lowerCamelCase							,		'''w'''							,		encoding='''utf-8'''     ) as f:
						f.write(__lowerCamelCase     )


# make sure we are under the root of the project
_lowercase			:			List[str]										=						Path(__file__).resolve().parent.parent.parent
_lowercase			:			Dict										=						repo_dir / "model_cards"

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
		_lowercase			:			int										=						model_cards_dir / "allenai" / model_name
		write_model_card(model_card_dir, src_lang="en", tgt_lang="de", model_name=model_name)




