





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class      lowerCAmelCase							(      __UpperCamelCase,						unittest.TestCase    ):
 UpperCAmelCase__									=	CLIPTokenizer
 UpperCAmelCase__									=	CLIPTokenizerFast
 UpperCAmelCase__									=	True
 UpperCAmelCase__									=	{}
 UpperCAmelCase__									=	False
 def   A_			(   self						:   Union[str, Any]			)    ->			List[Any]:
     super().setUp()

     # fmt: off
     lowerCamelCase__					:   str      				=	['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
     # fmt: on
     lowerCamelCase__					:   Union[str, Any]      				=	dict(zip(UpperCAmelCase    ,  range(len(UpperCAmelCase			)			)			)			)
     lowerCamelCase__					:   str      				=	['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
     lowerCamelCase__					:   int      				=	{'unk_token': '<unk>'}

     lowerCamelCase__					:   int      				=	os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES['vocab_file']			)
     lowerCamelCase__					:   List[Any]      				=	os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES['merges_file']			)
     with open(self.vocab_file    ,  'w'    ,  encoding='utf-8'			) as fp:
         fp.write(json.dumps(UpperCAmelCase			) + '\n'			)
     with open(self.merges_file    ,  'w'    ,  encoding='utf-8'			) as fp:
         fp.write('\n'.join(UpperCAmelCase			)			)
 def   A_			(   self						:   Union[str, Any]    ,  **UpperCAmelCase						:   Union[str, Any]			)    ->			Dict:
     kwargs.update(self.special_tokens_map			)
     return CLIPTokenizer.from_pretrained(self.tmpdirname    ,  **UpperCAmelCase			)
 def   A_			(   self						:   str    ,  **UpperCAmelCase						:   List[str]			)    ->			Optional[Any]:
     kwargs.update(self.special_tokens_map			)
     return CLIPTokenizerFast.from_pretrained(self.tmpdirname    ,  **UpperCAmelCase			)
 def   A_			(   self						:   Optional[int]    ,  UpperCAmelCase						:   Union[str, Any]			)    ->			List[str]:
     lowerCamelCase__					:   Union[str, Any]      				=	'lower newer'
     lowerCamelCase__					:   str      				=	'lower newer'
     return input_text, output_text
 def   A_			(   self						:   int			)    ->			Dict:
     lowerCamelCase__					:   str      				=	CLIPTokenizer(self.vocab_file    ,  self.merges_file    ,  **self.special_tokens_map			)
     lowerCamelCase__					:   Optional[int]      				=	'lower newer'
     lowerCamelCase__					:   List[Any]      				=	['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
     lowerCamelCase__					:   Tuple      				=	tokenizer.tokenize(UpperCAmelCase			)
     self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

     lowerCamelCase__					:   List[Any]      				=	tokens + [tokenizer.unk_token]
     lowerCamelCase__					:   List[str]      				=	[10, 2, 16, 9, 3, 2, 16, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase			)    ,  UpperCAmelCase			)
 @require_ftfy
 def   A_			(   self						:   str			)    ->			Optional[Any]:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
             lowerCamelCase__					:   List[Any]      				=	self.tokenizer_class.from_pretrained(UpperCAmelCase    ,  **UpperCAmelCase			)
             lowerCamelCase__					:   Tuple      				=	self.rust_tokenizer_class.from_pretrained(UpperCAmelCase    ,  **UpperCAmelCase			)

             lowerCamelCase__					:   Optional[Any]      				=	'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
             lowerCamelCase__					:   str      				=	tokenizer_s.tokenize(UpperCAmelCase			)
             lowerCamelCase__					:   Any      				=	tokenizer_r.tokenize(UpperCAmelCase			)

             self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

             # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
             # with Tilde) encoded in 2 different ways
             lowerCamelCase__					:   Tuple      				=	'xa\u0303y' + ' ' + 'x\xe3y'
             lowerCamelCase__					:   int      				=	tokenizer_s.tokenize(UpperCAmelCase			)
             lowerCamelCase__					:   int      				=	tokenizer_r.tokenize(UpperCAmelCase			)

             self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

             # Test that the tokenization is identical on unicode of space type
             lowerCamelCase__					:   Any      				=	[
                 '\u0009',  # (horizontal tab, '\t')
                 '\u000B',  # (vertical tab)
                 '\u000C',  # (form feed)
                 '\u0020',  # (space, ' ')
                 '\u200E',  # (left-to-right mark):w
                 '\u200F',  # (right-to-left mark)
             ]
             for unicode_seq in spaces_unicodes:
                 lowerCamelCase__					:   str      				=	tokenizer_s.tokenize(UpperCAmelCase			)
                 lowerCamelCase__					:   Optional[Any]      				=	tokenizer_r.tokenize(UpperCAmelCase			)

                 self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

             # Test that the tokenization is identical on unicode of line break type
             lowerCamelCase__					:   int      				=	[
                 '\u000A',  # (line feed, '\n')
                 '\r\n',  # (carriage return and line feed, '\r\n')
                 '\u000D',  # (carriage return, '\r')
                 '\r',  # (carriage return, '\r')
                 '\u000D',  # (carriage return, '\r')
                 '\u2028',  # (line separator)
                 '\u2029',  # (paragraph separator)
                 # "\u0085", # (next line)
             ]

             # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
             # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
             # space (and thus into an empty list).

             for unicode_seq in line_break_unicodes:
                 lowerCamelCase__					:   Optional[Any]      				=	tokenizer_s.tokenize(UpperCAmelCase			)
                 lowerCamelCase__					:   int      				=	tokenizer_r.tokenize(UpperCAmelCase			)

                 self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)
 def   A_			(   self						:   Optional[int]			)    ->			int:
     # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
             lowerCamelCase__					:   Tuple      				=	'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
             lowerCamelCase__					:   str      				=	F"""{text_of_1_token} {text_of_1_token}"""

             lowerCamelCase__					:   Union[str, Any]      				=	self.rust_tokenizer_class.from_pretrained(
                 UpperCAmelCase    ,  use_fast=UpperCAmelCase    ,  )
             lowerCamelCase__					:   int      				=	tokenizer_r(UpperCAmelCase    ,  return_offsets_mapping=UpperCAmelCase    ,  add_special_tokens=UpperCAmelCase			)
             self.assertEqual(encoding.offset_mapping[0]    ,  (0, len(UpperCAmelCase			))			)
             self.assertEqual(
                 encoding.offset_mapping[1]    ,  (len(UpperCAmelCase			) + 1, len(UpperCAmelCase			) + 1 + len(UpperCAmelCase			))    ,  )

             lowerCamelCase__					:   Optional[Any]      				=	F""" {text}"""

             lowerCamelCase__					:   List[str]      				=	self.rust_tokenizer_class.from_pretrained(
                 UpperCAmelCase    ,  use_fast=UpperCAmelCase    ,  )
             lowerCamelCase__					:   Optional[Any]      				=	tokenizer_r(UpperCAmelCase    ,  return_offsets_mapping=UpperCAmelCase    ,  add_special_tokens=UpperCAmelCase			)
             self.assertEqual(encoding.offset_mapping[0]    ,  (1, 1 + len(UpperCAmelCase			))			)
             self.assertEqual(
                 encoding.offset_mapping[1]    ,  (1 + len(UpperCAmelCase			) + 1, 1 + len(UpperCAmelCase			) + 1 + len(UpperCAmelCase			))    ,  )
 def   A_			(   self						:   Any			)    ->			Union[str, Any]:
     # Test related to the breaking change introduced in transformers v4.17.0
     # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
     with self.assertRaises(UpperCAmelCase			) as context:
         self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'			)

     self.assertTrue(
         context.exception.args[0].startswith(
             'The `backend_tokenizer` provided does not match the expected format.'			)			)
 @require_ftfy
 def   A_			(   self						:   Any			)    ->			Tuple:
     super().test_tokenization_python_rust_equals()







 def   A_			(   self						:   Tuple			)    ->			str:
     # CLIP always lower cases letters
     pass












import dataclasses
import re
import string
from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple

import numpy as np

from . import residue_constants


_UpperCAmelCase       : int			     =       Mapping[str, np.ndarray]
_UpperCAmelCase       : List[Any]			     =       Mapping[str, Any]  # Is a nested dict.
_UpperCAmelCase       : Dict			     =       0.01

@dataclasses.dataclass(frozen=__UpperCamelCase    )
class      lowerCAmelCase							:
 UpperCAmelCase__									=	42  # [num_res, num_atom_type, 3]

 # Amino-acid type for each residue represented as an integer between 0 and
 # 20, where 20 is 'X'.
 UpperCAmelCase__									=	42  # [num_res]

 # Binary float mask to indicate presence of a particular atom. 1.0 if an atom
 # is present and 0.0 if not. This should be used for loss masking.
 UpperCAmelCase__									=	42  # [num_res, num_atom_type]

 # Residue index as used in PDB. It is not necessarily continuous or 0-indexed.
 UpperCAmelCase__									=	42  # [num_res]

 # B-factors, or temperature factors, of each residue (in sq. angstroms units),
 # representing the displacement of the residue from its ground truth mean
 # value.
 UpperCAmelCase__									=	42  # [num_res, num_atom_type]

 # Chain indices for multi-chain predictions
 UpperCAmelCase__									=	None

 # Optional remark about the protein. Included as a comment in output PDB
 # files
 UpperCAmelCase__									=	None

 # Templates used to generate this protein (prediction-only)
 UpperCAmelCase__									=	None

 # Chain corresponding to each parent
 UpperCAmelCase__									=	None


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase				)    ->  Protein:
    lowerCamelCase__					:   Optional[int]      				=	r'(\[[A-Z]+\]\n)'
    lowerCamelCase__					:   List[str]      				=	[tag.strip() for tag in re.split(_UpperCAmelCase		,			_UpperCAmelCase				) if len(_UpperCAmelCase				) > 0]
    lowerCamelCase__					:   Iterator[Tuple[str, List[str]]]      				=	zip(tags[0::2]		,			[l.split('\n'				) for l in tags[1::2]]				)

    lowerCamelCase__					:   List[str]      				=	["N", "CA", "C"]
    lowerCamelCase__					:   Dict      				=	None
    lowerCamelCase__					:   str      				=	None
    lowerCamelCase__					:   int      				=	None
    for g in groups:
        if "[PRIMARY]" == g[0]:
            lowerCamelCase__					:   int      				=	g[1][0].strip()
            for i in range(len(_UpperCAmelCase				)				):
                if seq[i] not in residue_constants.restypes:
                    lowerCamelCase__					:   Union[str, Any]      				=	'X'  # FIXME: strings are immutable
            lowerCamelCase__					:   Union[str, Any]      				=	np.array(
                [residue_constants.restype_order.get(_UpperCAmelCase		,			residue_constants.restype_num				) for res_symbol in seq]				)
        elif "[TERTIARY]" == g[0]:
            lowerCamelCase__					:   List[List[float]]      				=	[]
            for axis in range(3				):
                tertiary.append(list(map(_UpperCAmelCase		,			g[1][axis].split()				)				)				)
            lowerCamelCase__					:   int      				=	np.array(_UpperCAmelCase				)
            lowerCamelCase__					:   Tuple      				=	np.zeros((len(tertiary[0]				) // 3, residue_constants.atom_type_num, 3)				).astype(np.floataa				)
            for i, atom in enumerate(_UpperCAmelCase				):
                lowerCamelCase__					:   int      				=	np.transpose(tertiary_np[:, i::3]				)
            atom_positions *= PICO_TO_ANGSTROM
        elif "[MASK]" == g[0]:
            lowerCamelCase__					:   int      				=	np.array(list(map({'-': 0, '+': 1}.get		,			g[1][0].strip()				)				)				)
            lowerCamelCase__					:   List[Any]      				=	np.zeros(
                (
                    len(_UpperCAmelCase				),
                    residue_constants.atom_type_num,
                )				).astype(np.floataa				)
            for i, atom in enumerate(_UpperCAmelCase				):
                lowerCamelCase__					:   Union[str, Any]      				=	1
            atom_mask *= mask[..., None]

    assert aatype is not None

    return Protein(
        atom_positions=_UpperCAmelCase		,			atom_mask=_UpperCAmelCase		,			aatype=_UpperCAmelCase		,			residue_index=np.arange(len(_UpperCAmelCase				)				)		,			b_factors=_UpperCAmelCase		,			)


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase = 0				)    ->  List[str]:
    lowerCamelCase__					:   List[str]      				=	[]

    lowerCamelCase__					:   Dict      				=	prot.remark
    if remark is not None:
        pdb_headers.append(F"""REMARK {remark}"""				)

    lowerCamelCase__					:   str      				=	prot.parents
    lowerCamelCase__					:   Union[str, Any]      				=	prot.parents_chain_index
    if parents is not None and parents_chain_index is not None:
        lowerCamelCase__					:   Any      				=	[p for i, p in zip(_UpperCAmelCase		,			_UpperCAmelCase				) if i == chain_id]

    if parents is None or len(_UpperCAmelCase				) == 0:
        lowerCamelCase__					:   List[Any]      				=	['N/A']

    pdb_headers.append(F"""PARENT {" ".join(_UpperCAmelCase				)}"""				)

    return pdb_headers


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase				)    ->  str:
    lowerCamelCase__					:   List[str]      				=	[]
    lowerCamelCase__					:   str      				=	pdb_str.split('\n'				)

    lowerCamelCase__					:   int      				=	prot.remark
    if remark is not None:
        out_pdb_lines.append(F"""REMARK {remark}"""				)

    lowerCamelCase__					:   List[List[str]]
    if prot.parents is not None and len(prot.parents				) > 0:
        lowerCamelCase__					:   List[Any]      				=	[]
        if prot.parents_chain_index is not None:
            lowerCamelCase__					:   Dict[str, List[str]]      				=	{}
            for p, i in zip(prot.parents		,			prot.parents_chain_index				):
                parent_dict.setdefault(str(_UpperCAmelCase				)		,			[]				)
                parent_dict[str(_UpperCAmelCase				)].append(_UpperCAmelCase				)

            lowerCamelCase__					:   str      				=	max([int(_UpperCAmelCase				) for chain_idx in parent_dict]				)
            for i in range(max_idx + 1				):
                lowerCamelCase__					:   Optional[Any]      				=	parent_dict.get(str(_UpperCAmelCase				)		,			['N/A']				)
                parents_per_chain.append(_UpperCAmelCase				)
        else:
            parents_per_chain.append(list(prot.parents				)				)
    else:
        lowerCamelCase__					:   Union[str, Any]      				=	[['N/A']]

    def make_parent_line(_UpperCAmelCase				) -> str:
        return F"""PARENT {" ".join(_UpperCAmelCase				)}"""

    out_pdb_lines.append(make_parent_line(parents_per_chain[0]				)				)

    lowerCamelCase__					:   List[Any]      				=	0
    for i, l in enumerate(_UpperCAmelCase				):
        if "PARENT" not in l and "REMARK" not in l:
            out_pdb_lines.append(_UpperCAmelCase				)
        if "TER" in l and "END" not in lines[i + 1]:
            chain_counter += 1
            if not chain_counter >= len(_UpperCAmelCase				):
                lowerCamelCase__					:   Union[str, Any]      				=	parents_per_chain[chain_counter]
            else:
                lowerCamelCase__					:   Optional[Any]      				=	['N/A']

            out_pdb_lines.append(make_parent_line(_UpperCAmelCase				)				)

    return "\n".join(_UpperCAmelCase				)


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase				)    ->  str:
    lowerCamelCase__					:   Tuple      				=	residue_constants.restypes + ['X']

    def res_atoa(_UpperCAmelCase				) -> str:
        return residue_constants.restype_atoa.get(restypes[r]		,			'UNK'				)

    lowerCamelCase__					:   int      				=	residue_constants.atom_types

    lowerCamelCase__					:   List[str]      				=	[]

    lowerCamelCase__					:   Union[str, Any]      				=	prot.atom_mask
    lowerCamelCase__					:   Union[str, Any]      				=	prot.aatype
    lowerCamelCase__					:   int      				=	prot.atom_positions
    lowerCamelCase__					:   List[Any]      				=	prot.residue_index.astype(np.intaa				)
    lowerCamelCase__					:   Optional[int]      				=	prot.b_factors
    lowerCamelCase__					:   Any      				=	prot.chain_index

    if np.any(aatype > residue_constants.restype_num				):
        raise ValueError('Invalid aatypes.'				)

    lowerCamelCase__					:   List[Any]      				=	get_pdb_headers(_UpperCAmelCase				)
    if len(_UpperCAmelCase				) > 0:
        pdb_lines.extend(_UpperCAmelCase				)

    lowerCamelCase__					:   Union[str, Any]      				=	aatype.shape[0]
    lowerCamelCase__					:   Optional[Any]      				=	1
    lowerCamelCase__					:   str      				=	0
    lowerCamelCase__					:   Tuple      				=	string.ascii_uppercase
    lowerCamelCase__					:   str      				=	None
    # Add all atom sites.
    for i in range(_UpperCAmelCase				):
        lowerCamelCase__					:   List[Any]      				=	res_atoa(aatype[i]				)
        for atom_name, pos, mask, b_factor in zip(_UpperCAmelCase		,			atom_positions[i]		,			atom_mask[i]		,			b_factors[i]				):
            if mask < 0.5:
                continue

            lowerCamelCase__					:   Union[str, Any]      				=	'ATOM'
            lowerCamelCase__					:   Optional[int]      				=	atom_name if len(_UpperCAmelCase				) == 4 else F""" {atom_name}"""
            lowerCamelCase__					:   Any      				=	''
            lowerCamelCase__					:   Optional[Any]      				=	''
            lowerCamelCase__					:   str      				=	1.00
            lowerCamelCase__					:   str      				=	atom_name[0]  # Protein supports only C, N, O, S, this works.
            lowerCamelCase__					:   List[str]      				=	''

            lowerCamelCase__					:   str      				=	'A'
            if chain_index is not None:
                lowerCamelCase__					:   List[Any]      				=	chain_tags[chain_index[i]]

            # PDB is a columnar format, every space matters here!
            lowerCamelCase__					:   Union[str, Any]      				=	(
                F"""{record_type:<6}{atom_index:>5} {name:<4}{alt_loc:>1}"""
                F"""{res_name_a:>3} {chain_tag:>1}"""
                F"""{residue_index[i]:>4}{insertion_code:>1}   """
                F"""{pos[0]:>8.3f}{pos[1]:>8.3f}{pos[2]:>8.3f}"""
                F"""{occupancy:>6.2f}{b_factor:>6.2f}          """
                F"""{element:>2}{charge:>2}"""
            )
            pdb_lines.append(_UpperCAmelCase				)
            atom_index += 1

        lowerCamelCase__					:   Dict      				=	i == n - 1
        if chain_index is not None:
            if i != n - 1 and chain_index[i + 1] != prev_chain_index:
                lowerCamelCase__					:   List[Any]      				=	True
                lowerCamelCase__					:   List[str]      				=	chain_index[i + 1]

        if should_terminate:
            # Close the chain.
            lowerCamelCase__					:   int      				=	'TER'
            lowerCamelCase__					:   Any      				=	(
                F"""{chain_end:<6}{atom_index:>5}      {res_atoa(aatype[i]				):>3} {chain_tag:>1}{residue_index[i]:>4}"""
            )
            pdb_lines.append(_UpperCAmelCase				)
            atom_index += 1

            if i != n - 1:
                # "prev" is a misnomer here. This happens at the beginning of
                # each new chain.
                pdb_lines.extend(get_pdb_headers(_UpperCAmelCase		,			_UpperCAmelCase				)				)

    pdb_lines.append('END'				)
    pdb_lines.append(''				)
    return "\n".join(_UpperCAmelCase				)


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase				)    ->  np.ndarray:
    return residue_constants.STANDARD_ATOM_MASK[prot.aatype]


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase = None		,			_UpperCAmelCase = None		,			_UpperCAmelCase = None		,			_UpperCAmelCase = None		,			_UpperCAmelCase = None		,			)    ->  Protein:
    return Protein(
        aatype=features['aatype']		,			atom_positions=result['final_atom_positions']		,			atom_mask=result['final_atom_mask']		,			residue_index=features['residue_index'] + 1		,			b_factors=b_factors if b_factors is not None else np.zeros_like(result['final_atom_mask']				)		,			chain_index=_UpperCAmelCase		,			remark=_UpperCAmelCase		,			parents=_UpperCAmelCase		,			parents_chain_index=_UpperCAmelCase		,			)





