code: def test_hipe_2022_corpus(tasks_base_path):
      # This test covers the complete HIPE 2022 dataset.
      # https://github.com/hipe-eval/HIPE-2022-data
      # Includes variant with document separator, and all versions of the dataset.
  
      # We have manually checked, that these numbers are correct:
      hipe_stats = {
          "v1.0": {
              "ajmc": {
                  "de": {
                      "sample": {
                          "sents": 119,
                          "docs": 8,
                          "labels": [
                              "date",
                              "loc",
                              "pers",
                              "scope",
                              "work",
                          ],
                      }
                  },
                  "en": {
                      "sample": {
                          "sents": 83,
                          "docs": 5,
                          "labels": [
                              "date",
                              "loc",
                              "pers",
                              "scope",
                              "work",
                          ],
                      }
                  },
              },
              "hipe2020": {
                  "de": {
                      "train": {
                          "sents": 3470
                          + 2,  # 2 sentences with missing EOS marker
                          "docs": 103,
                          "labels": [
                              "loc",
                              "org",
                              "pers",
                              "prod",
                              "time",
                          ],
                      },
                      "dev": {
                          "sents": 1202,
                          "docs": 33,
                          "labels": [
                              "loc",
                              "org",
                              "pers",
                              "prod",
                              "time",
                          ],
                      },
                  },
                  "en": {
                      "dev": {
                          "sents": 1045,
                          "docs": 80,
                          "labels": [
                              "loc",
                              "org",
                              "pers",
                              "prod",
                              "time",
                          ],
                      }
                  },
                  "fr": {
                      "train": {
                          "sents": 5743,
                          "docs": 158,
                          "labels": [
                              "loc",
                              "org",
                              "pers",
                              "prod",
                              "time",
                              "comp",
                          ],
                      },
                      "dev": {
                          "sents": 1244,
                          "docs": 43,
                          "labels": [
                              "loc",
                              "org",
                              "pers",
                              "prod",
                              "time",
                          ],
                      },
                  },
              },
              "letemps": {
                  "fr": {
                      "train": {
                          "sents": 14051,
                          "docs": 414,
                          "labels": ["loc", "org", "pers"],
                      },
                      "dev": {
                          "sents": 1341,
                          "docs": 51,
                          "labels": ["loc", "org", "pers"],
                      },
                  }
              },
              "newseye": {
                  # +1 offset, because of missing EOS marker at EOD
                  "de": {
                      "train": {
                          "sents": 23646 + 1,
                          "docs": 11,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev": {
                          "sents": 1110 + 1,
                          "docs": 12,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev2": {
                          "sents": 1541 + 1,
                          "docs": 12,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                  },
                  "fi": {
                      "train": {
                          "sents": 1141 + 1,
                          "docs": 24,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev": {
                          "sents": 140 + 1,
                          "docs": 24,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev2": {
                          "sents": 104 + 1,
                          "docs": 21,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                  },
                  "fr": {
                      "train": {
                          "sents": 7106 + 1,
                          "docs": 35,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev": {
                          "sents": 662 + 1,
                          "docs": 35,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev2": {
                          "sents": 1016 + 1,
                          "docs": 35,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                  },
                  "sv": {
                      "train": {
                          "sents": 1063 + 1,
                          "docs": 21,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev": {
                          "sents": 126 + 1,
                          "docs": 21,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                      "dev2": {
                          "sents": 136 + 1,
                          "docs": 21,
                          "labels": ["HumanProd", "LOC", "ORG", "PER"],
                      },
                  },
              },
              "sonar": {
                  "de": {
                      "dev": {
                          "sents": 1603
                          + 10,  # 10 sentences with missing EOS marker
                          "docs": 10,
                          "labels": ["LOC", "ORG", "PER"],
                      }
                  }
              },
              "topres19th": {
                  "en": {
                      "train": {
                          "sents": 5874,
                          "docs": 309,
                          "labels": ["BUILDING", "LOC", "STREET"],
                      },
                      "dev": {
                          "sents": 646,
                          "docs": 34,
                          "labels": ["BUILDING", "LOC", "STREET"],
                      },
                  }
              },
          }
      }
  
      hipe_stats["v2.0"] = copy.deepcopy(hipe_stats["v1.0"])
      hipe_stats["v2.0"]["ajmc"] = {
          "de": {
              "train": {
                  "sents": 1022
                  + 2,  # 2 sentences with missing EOS marker
                  "docs": 76,
                  "labels": [
                      "date",
                      "loc",
                      "object",
                      "pers",
                      "scope",
                      "work",
                  ],
              },
              "dev": {
                  "sents": 192,
                  "docs": 14,
                  "labels": ["loc", "object", "pers", "scope", "work"],
              },
          },
          "en": {
              "train": {
                  "sents": 1153
                  + 1,  # 1 sentence with missing EOS marker
                  "docs": 60,
                  "labels": [
                      "date",
                      "loc",
                      "object",
                      "pers",
                      "scope",
                      "work",
                  ],
              },
              "dev": {
                  "sents": 251
                  + 1,  # 1 sentence with missing EOS marker
                  "docs": 14,
                  "labels": ["date", "loc", "pers", "scope", "work"],
              },
          },
          "fr": {
              "train": {
                  "sents": 893
                  + 1,  # 1 sentence with missing EOS marker
                  "docs": 72,
                  "labels": [
                      "date",
                      "loc",
                      "object",
                      "pers",
                      "scope",
                      "work",
                  ],
              },
              "dev": {
                  "sents": 201
                  + 1,  # 1 sentence with missing EOS marker
                  "docs": 17,
                  "labels": ["pers", "scope", "work"],
              },
          },
      }
      hipe_stats["v2.0"]["newseye"]["de"] = {
          "train": {
              "sents": 20839 + 1,
              "docs": 7,
              "labels": ["HumanProd", "LOC", "ORG", "PER"],
          }  # missing EOD marker
      }
      hipe_stats["v2.0"]["sonar"] = {
          "de": {
              "dev": {
                  "sents": 816
                  + 10,  # 9 sentences with missing EOS marker + missing EOD
                  "docs": 10,
                  "labels": ["LOC", "ORG", "PER"],
              }
          }
      }
      hipe_stats["v2.1"] = copy.deepcopy(hipe_stats["v2.0"])
      hipe_stats["v2.1"]["hipe2020"]["fr"]["train"] = {
          "sents": 5743,
          "docs": 158,
          "labels": ["loc", "org", "pers", "prod", "time"],
      }
  
      # Test data for v2.1 release
      hipe_stats["v2.1"]["ajmc"]["de"]["test"] = {
          "sents": 224,
          "docs": 16,
          "labels": ["loc", "object", "pers", "scope", "work"],
      }
      hipe_stats["v2.1"]["ajmc"]["en"]["test"] = {
          "sents": 238,
          "docs": 13,
          "labels": ["date", "loc", "pers", "scope", "work"],
      }
      hipe_stats["v2.1"]["ajmc"]["fr"]["test"] = {
          "sents": 188 + 1,  # 1 sentence with missing EOS marker
          "docs": 15,
          "labels": ["date", "loc", "pers", "scope", "work"],
      }
      hipe_stats["v2.1"]["hipe2020"]["de"]["test"] = {
          "sents": 1215 + 2,  # 2 sentences with missing EOS marker
          "docs": 49,
          "labels": ["loc", "org", "pers", "prod", "time"],
      }
      hipe_stats["v2.1"]["hipe2020"]["en"]["test"] = {
          "sents": 553,
          "docs": 46,
          "labels": ["loc", "org", "pers", "prod", "time"],
      }
      hipe_stats["v2.1"]["hipe2020"]["fr"]["test"] = {
          "sents": 1462,
          "docs": 43,
          "labels": ["loc", "org", "pers", "prod", "time"],
      }
      hipe_stats["v2.1"]["letemps"]["fr"]["test"] = {
          "sents": 2381,
          "docs": 51,
          "labels": ["loc", "org", "pers"],
      }
      hipe_stats["v2.1"]["newseye"]["de"]["test"] = {
          "sents": 3336 + 1,  # 1 missing EOD marker
          "docs": 13,
          "labels": ["HumanProd", "LOC", "ORG", "PER"],
      }
      hipe_stats["v2.1"]["newseye"]["fi"]["test"] = {
          "sents": 390 + 1,  # 1 missing EOD marker
          "docs": 24,
          "labels": ["HumanProd", "LOC", "ORG", "PER"],
      }
      hipe_stats["v2.1"]["newseye"]["fr"]["test"] = {
          "sents": 2534 + 1,  # 1 missing EOD marker
          "docs": 35,
          "labels": ["HumanProd", "LOC", "ORG", "PER"],
      }
      hipe_stats["v2.1"]["newseye"]["sv"]["test"] = {
          "sents": 342 + 1,  # 1 missing EOD marker
          "docs": 21,
          "labels": ["HumanProd", "LOC", "ORG", "PER"],
      }
      hipe_stats["v2.1"]["sonar"]["de"]["test"] = {
          "sents": 807 + 8 + 1,  # 8 missing EOS marker + missing EOD
          "docs": 10,
          "labels": ["LOC", "ORG", "PER"],
      }
      hipe_stats["v2.1"]["topres19th"]["en"]["test"] = {
          "sents": 2001,
          "docs": 112,
          "labels": ["BUILDING", "LOC", "STREET"],
      }
  
      def test_hipe_2022(
          dataset_version="v2.1", add_document_separator=True
      ):
          for dataset_name, languages in hipe_stats[
              dataset_version
          ].items():
              for language in languages:
                  splits = languages[language]
  
                  corpus = flair.datasets.NER_HIPE_2022(
                      version=dataset_version,
                      dataset_name=dataset_name,
                      language=language,
                      dev_split_name="dev",
                      add_document_separator=add_document_separator,
                  )
  
                  for split_name, stats in splits.items():
                      split_description = f"{dataset_name}@{dataset_version}/{language}#{split_name}"
  
                      current_sents = stats["sents"]
                      current_docs = stats["docs"]
                      current_labels = set(stats["labels"])
  
                      total_sentences = (
                          current_sents + current_docs
                          if add_document_separator
                          else stats["sents"]
                      )
  
                      if split_name == "train":
                          assert (
                              len(corpus.train) == total_sentences
                          ), f"Sentence count mismatch for {split_description}: {len(corpus.train)} vs. {total_sentences}"
  
                          gold_labels = set(
                              corpus.make_label_dictionary(
                                  label_type="ner"
                              ).get_items()
                          )
  
                          assert (
                              current_labels == gold_labels
                          ), f"Label mismatch for {split_description}: {current_labels} vs. {gold_labels}"
  
                      elif split_name in ["dev", "sample"]:
                          assert (
                              len(corpus.dev) == total_sentences
                          ), f"Sentence count mismatch for {split_description}: {len(corpus.dev)} vs. {total_sentences}"
  
                          corpus._train = corpus._dev
                          gold_labels = set(
                              corpus.make_label_dictionary(
                                  label_type="ner"
                              ).get_items()
                          )
  
                          assert (
                              current_labels == gold_labels
                          ), f"Label mismatch for {split_description}: {current_labels} vs. {gold_labels}"
                      elif split_name == "dev2":
                          corpus = flair.datasets.NER_HIPE_2022(
                              version=dataset_version,
                              dataset_name=dataset_name,
                              language=language,
                              dev_split_name="dev2",
                              add_document_separator=add_document_separator,
                          )
  
                          corpus._train = corpus._dev
                          gold_labels = set(
                              corpus.make_label_dictionary(
                                  label_type="ner"
                              ).get_items()
                          )
  
                          assert (
                              len(corpus.dev) == total_sentences
                          ), f"Sentence count mismatch for {split_description}: {len(corpus.dev)} vs. {total_sentences}"
  
                          assert (
                              current_labels == gold_labels
                          ), f"Label mismatch for {split_description}: {current_labels} vs. {gold_labels}"
  
      test_hipe_2022(
          dataset_version="v1.0", add_document_separator=True
      )
      test_hipe_2022(
          dataset_version="v1.0", add_document_separator=False
      )
      test_hipe_2022(
          dataset_version="v2.0", add_document_separator=True
      )
      test_hipe_2022(
          dataset_version="v2.0", add_document_separator=False
      )
      test_hipe_2022(
          dataset_version="v2.1", add_document_separator=True
      )
      test_hipe_2022(
          dataset_version="v2.1", add_document_separator=False
      )
  
imports: import copy
  import shutil
  import pytest
  import flair
  import flair.datasets
  from flair.data import MultiCorpus, Sentence
  from flair.datasets import ColumnCorpus
  from flair.datasets.sequence_labeling import ONTONOTES, JsonlCorpus, JsonlDataset, MultiFileJsonlCorpus
label_desc: Use image node_id 24 for calling a global function with example usage: test_hipe_2022_corpus(tasks_base_path) without return types