code: def process_bucket(
      bucket_file_path,
      processed_directory,
      move_dir,
      tqdm_func,
      global_tqdm,
  ):
      bucket_id = re.sub(
          "\D", "", os.path.basename(bucket_file_path)
      )  # noqa: W605
      done_file = os.path.join(
          processed_directory,
          f"ngram_bucket_processing_{bucket_id}.done",
      )
      if os.path.exists(done_file):
          logger.info(f"bucket {bucket_id} already processed, skipping")
          return
  
      # For managing tqdm
      file_size = os.path.getsize(bucket_file_path)
      bucket_progress = tqdm_func(
          total=file_size, dynamic_ncols=True, unit="byte", unit_scale=1
      )
      current_file_position = 0
      update_frequency = 100 * 1000000  # 100mb
      update_counter = 0
  
      # Iterate through and output ngrams which occur in more then 10 documents
      bucket = TextReader(bucket_file_path)
  
      output_file_path = bucket_file_path + ".processed"
      output_archive = TextArchive(output_file_path, mode="wb")
  
      current_ngram = ""
      current_ngram_document_ids = set()
      for line in bucket.read():
          [ngram, document_id] = line.rsplit(" ", 1)
  
          # Write ngram if more then 10 unique document occurrences
          if ngram != current_ngram:
              if len(current_ngram_document_ids) > 10:
                  output_archive.add_data(
                      f"{current_ngram} {len(current_ngram_document_ids)}"
                  )
              current_ngram = ngram
              current_ngram_document_ids = set()
  
          current_ngram_document_ids.add(document_id)
  
          # Update tqdm
          update_counter += bucket.fh.tell() - current_file_position
          current_file_position = bucket.fh.tell()
          if update_counter > update_frequency:
              bucket_progress.update(update_counter)
              update_counter = 0
  
      # Remainder
      if len(current_ngram_document_ids) > 10:
          output_archive.add_data(
              f"{current_ngram} {len(current_ngram_document_ids)}"
          )
  
      output_archive.commit()
      Path(done_file).touch()
  
      if move_dir:
          shutil.move(output_file_path, move_dir)
  
      global_tqdm.update()
  
file_path: lm-evaluation-harness/scripts/clean_training_data/process_sorted_buckets.py
imports: import argparse
  import glob
  import os
  from pathlib import Path
  import re
  import shutil
  from tqdm import tqdm
  from tqdm_multiprocess import TqdmMultiProcessPool
  from scripts.clean_training_data.archiver import TextReader, TextArchive
  import logging
  from tqdm_multiprocess.logger import setup_logger_tqdm
label_desc: Use image node_id 1 for calling a global function with example usage: process_bucket(bucket_file_path, processed_directory, move_dir, tqdm_func, global_tqdm) without return types