





import argparse
import json
from typing import List

from ltp import LTP

from transformers import BertTokenizer




def     a__ (					lowercase__  ):






   '''simple docstring'''






   if (
       (cp >= 0x4E_00 and cp <= 0x9F_FF)
       or (cp >= 0x34_00 and cp <= 0x4D_BF)  #
       or (cp >= 0x2_00_00 and cp <= 0x2_A6_DF)  #
       or (cp >= 0x2_A7_00 and cp <= 0x2_B7_3F)  #
       or (cp >= 0x2_B7_40 and cp <= 0x2_B8_1F)  #
       or (cp >= 0x2_B8_20 and cp <= 0x2_CE_AF)  #
       or (cp >= 0xF9_00 and cp <= 0xFA_FF)
       or (cp >= 0x2_F8_00 and cp <= 0x2_FA_1F)  #
   ):  #
      return True

   return False




def     a__ (					lowercase__  ):






   '''simple docstring'''






   for char in word:
      UpperCAmelCase_											=ord(_lowerCamelCase  )
      if not _is_chinese_char(_lowerCamelCase  ):
         return 0
   return 1




def     a__ (					lowercase__  ):






   '''simple docstring'''






   UpperCAmelCase_											=set()

   for token in tokens:
      UpperCAmelCase_											=len(_lowerCamelCase  ) > 1 and is_chinese(_lowerCamelCase  )
      if chinese_word:
         word_set.add(_lowerCamelCase  )
   UpperCAmelCase_											=list(_lowerCamelCase  )
   return word_list




def     a__ (					lowercase__  ,    lowercase__  ):






   '''simple docstring'''






   if not chinese_word_set:
      return bert_tokens
   UpperCAmelCase_											=max([len(_lowerCamelCase  ) for w in chinese_word_set]  )

   UpperCAmelCase_											=bert_tokens
   UpperCAmelCase_											=0, len(_lowerCamelCase  )
   while start < end:
      UpperCAmelCase_											=True
      if is_chinese(bert_word[start]  ):
         UpperCAmelCase_											=min(end - start  ,    _lowerCamelCase  )
         for i in range(_lowerCamelCase  ,    1  ,    -1  ):
            UpperCAmelCase_											=''.join(bert_word[start : start + i]  )
            if whole_word in chinese_word_set:
               for j in range(start + 1  ,    start + i  ):
                  UpperCAmelCase_											='##' + bert_word[j]
               UpperCAmelCase_											=start + i
               UpperCAmelCase_											=False
               break
      if single_word:
         start += 1
   return bert_word




def     a__ (					lowercase__  ,    lowercase__  ,    lowercase__  ):






   '''simple docstring'''






   UpperCAmelCase_											=[]

   for i in range(0  ,    len(_lowerCamelCase  )  ,    1_0_0  ):
      UpperCAmelCase_											=ltp_tokenizer.seg(lines[i : i + 1_0_0]  )[0]
      UpperCAmelCase_											=[get_chinese_word(_lowerCamelCase  ) for r in res]
      ltp_res.extend(_lowerCamelCase  )
   assert len(_lowerCamelCase  ) == len(_lowerCamelCase  )

   UpperCAmelCase_											=[]
   for i in range(0  ,    len(_lowerCamelCase  )  ,    1_0_0  ):
      UpperCAmelCase_											=bert_tokenizer(lines[i : i + 1_0_0]  ,    add_special_tokens=_lowerCamelCase  ,    truncation=_lowerCamelCase  ,    max_length=5_1_2  )
      bert_res.extend(res["input_ids"]  )
   assert len(_lowerCamelCase  ) == len(_lowerCamelCase  )

   UpperCAmelCase_											=[]
   for input_ids, chinese_word in zip(_lowerCamelCase  ,    _lowerCamelCase  ):
      UpperCAmelCase_											=[]
      for id in input_ids:
         UpperCAmelCase_											=bert_tokenizer._convert_id_to_token(_lowerCamelCase  )
         input_tokens.append(_lowerCamelCase  )
      UpperCAmelCase_											=add_sub_symbol(_lowerCamelCase  ,    _lowerCamelCase  )
      UpperCAmelCase_											=[]
      # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
      for i, token in enumerate(_lowerCamelCase  ):
         if token[:2] == "##":
            UpperCAmelCase_											=token[2:]
            # save chinese tokens' pos
            if len(_lowerCamelCase  ) == 1 and _is_chinese_char(ord(_lowerCamelCase  )  ):
               ref_id.append(_lowerCamelCase  )
      ref_ids.append(_lowerCamelCase  )

   assert len(_lowerCamelCase  ) == len(_lowerCamelCase  )

   return ref_ids




def     a__ (					lowercase__  ):






   '''simple docstring'''






   with open(args.file_name  ,    "r"  ,    encoding="utf-8"  ) as f:
      UpperCAmelCase_											=f.readlines()
   UpperCAmelCase_											=[line.strip() for line in data if len(_lowerCamelCase  ) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
   UpperCAmelCase_											=LTP(args.ltp  )  # faster in GPU device
   UpperCAmelCase_											=BertTokenizer.from_pretrained(args.bert  )

   UpperCAmelCase_											=prepare_ref(_lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase  )

   with open(args.save_path  ,    "w"  ,    encoding="utf-8"  ) as f:
      UpperCAmelCase_											=[json.dumps(_lowerCamelCase  ) + '\n' for ref in ref_ids]
      f.writelines(_lowerCamelCase  )


if __name__ == "__main__":
     __lowercase  :   Union[str, Any]						      =argparse.ArgumentParser(description="""prepare_chinese_ref""")
     parser.add_argument(
         """--file_name""",
         type=str,
         default="""./resources/chinese-demo.txt""",
         help="""file need process, same as training data in lm""",
     )
     parser.add_argument(
         """--ltp""", type=str, default="""./resources/ltp""", help="""resources for LTP tokenizer, usually a path"""
     )
     parser.add_argument("""--bert""", type=str, default="""./resources/robert""", help="""resources for Bert tokenizer""")
     parser.add_argument("""--save_path""", type=str, default="""./resources/ref.txt""", help="""path to save res""")

     __lowercase  :   Any						      =parser.parse_args()
     main(args)







import os
import tempfile
import unittest

import numpy as np

from diffusers.utils import is_flax_available
from diffusers.utils.testing_utils import require_flax, slow


if is_flax_available():
     import jax
     import jax.numpy as jnp
     from flax.jax_utils import replicate
     from flax.training.common_utils import shard

     from diffusers import FlaxDDIMScheduler, FlaxDiffusionPipeline, FlaxStableDiffusionPipeline

@require_flax
class      A       (	unittest.TestCase						):



      def 	lowerCAmelCase__    (	self:						str       )						->    Union[str, Any]:
         '''simple docstring'''

         with tempfile.TemporaryDirectory() as tmpdirname:
            # pipeline has Flax weights
            UpperCAmelCase_											=FlaxDiffusionPipeline.from_pretrained(
                "hf-internal-testing/tiny-stable-diffusion-pipe"		,  safety_checker=_lowerCAmelCase		,  cache_dir=_lowerCAmelCase       )

            UpperCAmelCase_											=[t[-1] for t in os.walk(os.path.join(_lowerCAmelCase		,  os.listdir(_lowerCAmelCase       )[0]		,  "snapshots"       )       )]
            UpperCAmelCase_											=[item for sublist in all_root_files for item in sublist]

            # None of the downloaded files should be a PyTorch file even if we have some here:
            # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_pytorch_model.bin
            assert not any(f.endswith(".bin"       ) for f in files       )

@slow
@require_flax
class      A       (	unittest.TestCase						):



      def 	lowerCAmelCase__    (	self:						Union[str, Any]       )						->    Tuple:
         '''simple docstring'''

         UpperCAmelCase_						,       UpperCAmelCase_											=FlaxStableDiffusionPipeline.from_pretrained(
             "hf-internal-testing/tiny-stable-diffusion-pipe"		,  safety_checker=_lowerCAmelCase       )

         UpperCAmelCase_											=(
             "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of"
             " field, close up, split lighting, cinematic"
         )

         UpperCAmelCase_											=jax.random.PRNGKey(0       )
         UpperCAmelCase_											=4

         UpperCAmelCase_											=jax.device_count()
         UpperCAmelCase_											=num_samples * [prompt]
         UpperCAmelCase_											=pipeline.prepare_inputs(_lowerCAmelCase       )

         # shard inputs and rng
         UpperCAmelCase_											=replicate(_lowerCAmelCase       )
         UpperCAmelCase_											=jax.random.split(_lowerCAmelCase		,  _lowerCAmelCase       )
         UpperCAmelCase_											=shard(_lowerCAmelCase       )

         UpperCAmelCase_											=pipeline(_lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  jit=_lowerCAmelCase       ).images

         assert images.shape == (num_samples, 1, 64, 64, 3)
         if jax.device_count() == 8:
            assert np.abs(np.abs(images[0, 0, :2, :2, -2:]		,  dtype=np.floataa       ).sum() - 4.1_51_47_45       ) < 1e-3
            assert np.abs(np.abs(_lowerCAmelCase		,  dtype=np.floataa       ).sum() - 4_99_47.8_75       ) < 5e-1

         UpperCAmelCase_											=pipeline.numpy_to_pil(np.asarray(images.reshape((num_samples,) + images.shape[-3:]       )       )       )
         assert len(_lowerCAmelCase       ) == num_samples



      def 	lowerCAmelCase__    (	self:						List[Any]       )						->    Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_						,       UpperCAmelCase_											=FlaxStableDiffusionPipeline.from_pretrained(
             "CompVis/stable-diffusion-v1-4"		,  revision="flax"		,  safety_checker=_lowerCAmelCase       )

         UpperCAmelCase_											=(
             "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of"
             " field, close up, split lighting, cinematic"
         )

         UpperCAmelCase_											=jax.random.PRNGKey(0       )
         UpperCAmelCase_											=50

         UpperCAmelCase_											=jax.device_count()
         UpperCAmelCase_											=num_samples * [prompt]
         UpperCAmelCase_											=pipeline.prepare_inputs(_lowerCAmelCase       )

         # shard inputs and rng
         UpperCAmelCase_											=replicate(_lowerCAmelCase       )
         UpperCAmelCase_											=jax.random.split(_lowerCAmelCase		,  _lowerCAmelCase       )
         UpperCAmelCase_											=shard(_lowerCAmelCase       )

         UpperCAmelCase_											=pipeline(_lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  jit=_lowerCAmelCase       ).images

         assert images.shape == (num_samples, 1, 512, 512, 3)
         if jax.device_count() == 8:
            assert np.abs((np.abs(images[0, 0, :2, :2, -2:]		,  dtype=np.floataa       ).sum() - 0.05_65_24_01)       ) < 1e-3
            assert np.abs((np.abs(_lowerCAmelCase		,  dtype=np.floataa       ).sum() - 2_38_38_08.2)       ) < 5e-1



      def 	lowerCAmelCase__    (	self:						int       )						->    Union[str, Any]:
         '''simple docstring'''

         UpperCAmelCase_						,       UpperCAmelCase_											=FlaxStableDiffusionPipeline.from_pretrained(
             "CompVis/stable-diffusion-v1-4"		,  revision="bf16"		,  dtype=jnp.bfloataa		,  safety_checker=_lowerCAmelCase       )

         UpperCAmelCase_											=(
             "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of"
             " field, close up, split lighting, cinematic"
         )

         UpperCAmelCase_											=jax.random.PRNGKey(0       )
         UpperCAmelCase_											=50

         UpperCAmelCase_											=jax.device_count()
         UpperCAmelCase_											=num_samples * [prompt]
         UpperCAmelCase_											=pipeline.prepare_inputs(_lowerCAmelCase       )

         # shard inputs and rng
         UpperCAmelCase_											=replicate(_lowerCAmelCase       )
         UpperCAmelCase_											=jax.random.split(_lowerCAmelCase		,  _lowerCAmelCase       )
         UpperCAmelCase_											=shard(_lowerCAmelCase       )

         UpperCAmelCase_											=pipeline(_lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  jit=_lowerCAmelCase       ).images

         assert images.shape == (num_samples, 1, 512, 512, 3)
         if jax.device_count() == 8:
            assert np.abs((np.abs(images[0, 0, :2, :2, -2:]		,  dtype=np.floataa       ).sum() - 0.04_00_39_06)       ) < 1e-3
            assert np.abs((np.abs(_lowerCAmelCase		,  dtype=np.floataa       ).sum() - 2_37_35_16.75)       ) < 5e-1



      def 	lowerCAmelCase__    (	self:						Optional[int]       )						->    Tuple:
         '''simple docstring'''

         UpperCAmelCase_						,       UpperCAmelCase_											=FlaxStableDiffusionPipeline.from_pretrained(
             "CompVis/stable-diffusion-v1-4"		,  revision="bf16"		,  dtype=jnp.bfloataa       )

         UpperCAmelCase_											=(
             "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of"
             " field, close up, split lighting, cinematic"
         )

         UpperCAmelCase_											=jax.random.PRNGKey(0       )
         UpperCAmelCase_											=50

         UpperCAmelCase_											=jax.device_count()
         UpperCAmelCase_											=num_samples * [prompt]
         UpperCAmelCase_											=pipeline.prepare_inputs(_lowerCAmelCase       )

         # shard inputs and rng
         UpperCAmelCase_											=replicate(_lowerCAmelCase       )
         UpperCAmelCase_											=jax.random.split(_lowerCAmelCase		,  _lowerCAmelCase       )
         UpperCAmelCase_											=shard(_lowerCAmelCase       )

         UpperCAmelCase_											=pipeline(_lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  jit=_lowerCAmelCase       ).images

         assert images.shape == (num_samples, 1, 512, 512, 3)
         if jax.device_count() == 8:
            assert np.abs((np.abs(images[0, 0, :2, :2, -2:]		,  dtype=np.floataa       ).sum() - 0.04_00_39_06)       ) < 1e-3
            assert np.abs((np.abs(_lowerCAmelCase		,  dtype=np.floataa       ).sum() - 2_37_35_16.75)       ) < 5e-1



      def 	lowerCAmelCase__    (	self:						Optional[Any]       )						->    str:
         '''simple docstring'''

         UpperCAmelCase_											=FlaxDDIMScheduler(
             beta_start=0.0_00_85		,  beta_end=0.0_12		,  beta_schedule="scaled_linear"		,  set_alpha_to_one=_lowerCAmelCase		,  steps_offset=1		,  )

         UpperCAmelCase_						,       UpperCAmelCase_											=FlaxStableDiffusionPipeline.from_pretrained(
             "CompVis/stable-diffusion-v1-4"		,  revision="bf16"		,  dtype=jnp.bfloataa		,  scheduler=_lowerCAmelCase		,  safety_checker=_lowerCAmelCase		,  )
         UpperCAmelCase_											=scheduler.create_state()

         UpperCAmelCase_											=scheduler_state

         UpperCAmelCase_											=(
             "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of"
             " field, close up, split lighting, cinematic"
         )

         UpperCAmelCase_											=jax.random.PRNGKey(0       )
         UpperCAmelCase_											=50

         UpperCAmelCase_											=jax.device_count()
         UpperCAmelCase_											=num_samples * [prompt]
         UpperCAmelCase_											=pipeline.prepare_inputs(_lowerCAmelCase       )

         # shard inputs and rng
         UpperCAmelCase_											=replicate(_lowerCAmelCase       )
         UpperCAmelCase_											=jax.random.split(_lowerCAmelCase		,  _lowerCAmelCase       )
         UpperCAmelCase_											=shard(_lowerCAmelCase       )

         UpperCAmelCase_											=pipeline(_lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  jit=_lowerCAmelCase       ).images

         assert images.shape == (num_samples, 1, 512, 512, 3)
         if jax.device_count() == 8:
            assert np.abs((np.abs(images[0, 0, :2, :2, -2:]		,  dtype=np.floataa       ).sum() - 0.0_45_04_39_45)       ) < 1e-3
            assert np.abs((np.abs(_lowerCAmelCase		,  dtype=np.floataa       ).sum() - 2_34_76_93.5)       ) < 5e-1






      def 	lowerCAmelCase__    (	self:						Optional[Any]       )						->    int:
         '''simple docstring'''

         UpperCAmelCase_											=(
             "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of"
             " field, close up, split lighting, cinematic"
         )

         UpperCAmelCase_											=jax.device_count()
         UpperCAmelCase_											=num_samples * [prompt]
         UpperCAmelCase_											=jax.random.split(jax.random.PRNGKey(0       )		,  _lowerCAmelCase       )

         UpperCAmelCase_						,       UpperCAmelCase_											=FlaxStableDiffusionPipeline.from_pretrained(
             "CompVis/stable-diffusion-v1-4"		,  revision="bf16"		,  dtype=jnp.bfloataa		,  safety_checker=_lowerCAmelCase		,  )

         UpperCAmelCase_											=replicate(_lowerCAmelCase       )
         UpperCAmelCase_											=pipeline.prepare_inputs(_lowerCAmelCase       )
         UpperCAmelCase_											=shard(_lowerCAmelCase       )
         UpperCAmelCase_											=pipeline(_lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  jit=_lowerCAmelCase       ).images
         assert images.shape == (num_samples, 1, 512, 512, 3)
         UpperCAmelCase_											=images[2, 0, 256, 10:17, 1]

         # With memory efficient attention
         UpperCAmelCase_						,       UpperCAmelCase_											=FlaxStableDiffusionPipeline.from_pretrained(
             "CompVis/stable-diffusion-v1-4"		,  revision="bf16"		,  dtype=jnp.bfloataa		,  safety_checker=_lowerCAmelCase		,  use_memory_efficient_attention=_lowerCAmelCase		,  )

         UpperCAmelCase_											=replicate(_lowerCAmelCase       )
         UpperCAmelCase_											=pipeline.prepare_inputs(_lowerCAmelCase       )
         UpperCAmelCase_											=shard(_lowerCAmelCase       )
         UpperCAmelCase_											=pipeline(_lowerCAmelCase		,  _lowerCAmelCase		,  _lowerCAmelCase		,  jit=_lowerCAmelCase       ).images
         assert images_eff.shape == (num_samples, 1, 512, 512, 3)
         UpperCAmelCase_											=images[2, 0, 256, 10:17, 1]

         # I checked the results visually and they are very similar. However, I saw that the max diff is `1` and the `sum`
         # over the 8 images is exactly `256`, which is very suspicious. Testing a random slice for now.
         assert abs(slice_eff - slice       ).max() < 1e-2




