code: def test_get_open_datasets():
      # HF changed things so don't get raw list of all datasets, so not have to filter, but can't do negative filter
      open_tags = [
          "license:Apache License 2.0",
          "license:mit",
          "license:apache",
          "license:apache2",
          "license:apache-2.0",
          "license:bsd",
          "license:bsd-2-clause",
          "license:bsd-3-clause",
          "license:bsd-3-clause-clear",
          "license:lgpl-2.1",
          "license:lgpl-3.0",
          "license:lgpl-lr",
          "license:lgpl",
          "license:openrail++",
          "license:openrail",
          "license:bigscience-bloom-rail-1.0",
          # 'license:agpl-3.0',
          "license:other",
          "license:unknown",
          # 'license:mpl-2.0',     # ok, but would have to include original copyright, license, source, copies in distribution
          # Attribution required:
          "license:odc-by",
          "license:cc-by-4.0",
          "license:cc-by-3.0",
          "license:cc-by-2.0",
          "license:cc-by-2.5",
          # 'license:cc-by-sa-4.0',  # would require same license
          "license:odbl",
          "license:pddl",
          "license:ms-pl",
          "license:zlib",
      ]
      # bad license: cc-by-nc-4.0
  
      from huggingface_hub import list_datasets
  
      datasets = flatten_list(
          [[x for x in list_datasets(filter=y)] for y in open_tags]
      )
      datasets += [x for x in list_datasets(author="openai")]
      # check all:
      all_license_tags = set(
          flatten_list(
              [[y for y in x.tags if "license" in y] for x in datasets]
          )
      )
      print(len(all_license_tags))
      open_datasets = [
          x
          for x in datasets
          if any([y in x.tags for y in open_tags])
          or "license:" not in str(x.tags)
      ]
      print("open_datasets", len(open_datasets))
      all_task_tags = set(
          flatten_list(
              [
                  [y for y in x.tags if "task" in y]
                  for x in open_datasets
              ]
          )
      )
      print("all_task_tags", len(all_task_tags))
      excluded_tags = [
          "image",
          "hate",
          "tabular",
          "table-",
          "classification",
          "retrieval",
          "translation",
          "identification",
          "object",
          "mask",
          "to-text",
          "face-detection",
          "audio",
          "voice",
          "reinforcement",
          "depth-est",
          "forecasting",
          "parsing",
          "visual",
          "speech",
          "multiple-choice",
          "slot-filling",
          "irds/argsme",
          "-scoring",
          "other",
          "graph-ml",
          "feature-extraction",
          "keyword-spotting",
          "coreference-resolution",
          "segmentation",
          "word-sense-disambiguation",
          "lemmatization",
      ]
      task_tags = [
          x.replace("task_categories:", "").replace("task_ids:", "")
          for x in all_task_tags
          if not any([y in x for y in excluded_tags])
      ]
      print("task_tags", len(task_tags))
      # str(x.tags) to catch any pattern match to anything in list
      open_tasked_datasets = [
          x
          for x in open_datasets
          if any(
              [
                  y in str([x for x in x.tags if "task" in x])
                  for y in task_tags
              ]
          )
          and not any(
              [
                  y in str([x for x in x.tags if "task" in x])
                  for y in excluded_tags
              ]
          )
          or "task_categories" not in str(x.tags)
          and "task_ids" not in str(x.tags)
      ]
      open_tasked_datasets = [
          x for x in open_tasked_datasets if not x.disabled
      ]
      open_tasked_datasets = [
          x for x in open_tasked_datasets if not x.gated
      ]
      open_tasked_datasets = [
          x for x in open_tasked_datasets if not x.private
      ]
      print("open_tasked_datasets", len(open_tasked_datasets))
      sizes = list(
          set(
              flatten_list(
                  [
                      [(y, x.id) for y in x.tags if "size" in y]
                      for x in open_tasked_datasets
                  ]
              )
          )
      )
      languages = list(
          set(
              flatten_list(
                  [
                      [(y, x.id) for y in x.tags if "language:" in y]
                      for x in open_tasked_datasets
                  ]
              )
          )
      )
      open_english_tasked_datasets = [
          x
          for x in open_tasked_datasets
          if "language:" not in str(x.tags)
          or "language:en" in str(x.tags)
      ]
      small_open_english_tasked_datasets = [
          x
          for x in open_english_tasked_datasets
          if "n<1K" in str(x.tags)
          or "1K<n<10K" in str(x.tags)
          or "1K0<n<100K" in str(x.tags)
          or "100K<n<1M" in str(x.tags)
          or "size_category" not in str(x.tags)
      ]
      # 'aeslc' : email_body, subject -> summarization?
      # load_dataset(open_tasked_datasets[0].id).data['train'].to_pandas()
      ids = [x.id for x in small_open_english_tasked_datasets]
  
      # sanity checks
      # https://bair.berkeley.edu/blog/2023/04/03/koala/
      assert "alespalla/chatbot_instruction_prompts" in ids
      assert "laion/OIG" in ids
      assert "openai/webgpt_comparisons" in ids
      assert "openai/summarize_from_feedback" in ids
      assert "Anthropic/hh-rlhf" in ids
  
      # useful but not allowed for commercial purposes:
      # https://huggingface.co/datasets/squad
  
      print("open_english_tasked_datasets: ", ids, flush=True)
  
      exclude_ids = [
          "allenai/nllb",  # translation only
          "hf-internal-testing/fixtures_image_utils",  # testing
          "allenai/c4",  # search-url
          "agemagician/uniref50",  # unknown
          "huggingface-course/documentation-images",  # images
          "smilegate-ai/kor_unsmile",  # korean
          "MohamedRashad/ChatGPT-prompts",  # ChatGPT/LearnGPT/https://www.emergentmind.com/
          "humarin/chatgpt-paraphrases",  # Paraphrase using ChatGPT
          "Jeska/vaccinchat",  # not useful
          "alespalla/chatbot_instruction_prompts",  # mixes alpaca
          "allenai/prosocial-dialog",
          # already exlucded, but wrongly in other datasets that say more permissive license
          "AlekseyKorshuk/persona-chat",  # low quality
          "bavard/personachat_truecased",  # low quality
          "adamlin/daily_dialog",  # medium quality conversations
          "adamlin/FewShotWoz",  # low quality
          "benjaminbeilharz/better_daily_dialog",  # low quality
          "benjaminbeilharz/daily_dialog_w_turn_templates",  # low
          "benjaminbeilharz/empathetic_dialogues_for_lm",  # low
          "GEM-submissions/GEM__bart_base_schema_guided_dialog__1645547915",  # NA
          "ia-bentebib/conv_ai_2_fr",  # low fr
          "ia-bentebib/daily_dialog_fr",  # low fr
          "ia-bentebib/dialog_re_fr",  # low fr
          "ia-bentebib/empathetic_dialogues_fr",  # low fr
          "roskoN/dailydialog",  # low
          "VadorMazer/skyrimdialogstest",  # low
          "bigbio/med_qa",  # med specific Q/A
          "biu-nlp/qa_srl2018",  # low quality Q/A
          "biu-nlp/qa_discourse",  # low quality Q/A
          "iarfmoose/qa_evaluator",  # low quality Q/A
          "jeopardy",  # low quality Q/A -- no reasoning
          "narrativeqa",  # low quality Q/A
          "nomic-ai/gpt4all_prompt_generations",  # bad license
          "nomic-ai/gpt4all_prompt_generations_with_p3",  # bad license
          "HuggingFaceH4/alpaca",  # bad license
          "tatsu-lab/alpaca",  # ToS breaking
          "yahma/alpaca-cleaned",  # ToS breaking
          "Hello-SimpleAI/HC3",  # bad license
          "glue",  # no reasoning QA
          "sahil2801/CodeAlpaca-20k",  # bad license
          "Short-Answer-Feedback/saf_communication_networks_english",  # long Q, medium A
      ]
      small_open_english_tasked_datasets = [
          x
          for x in small_open_english_tasked_datasets
          if x.id not in exclude_ids
      ]
      # some ids clearly speech related
      small_open_english_tasked_datasets = [
          x
          for x in small_open_english_tasked_datasets
          if "speech" not in x.id
      ]
      # HF testing
      small_open_english_tasked_datasets = [
          x
          for x in small_open_english_tasked_datasets
          if "hf-internal-testing" not in x.id
      ]
      small_open_english_tasked_datasets = [
          x
          for x in small_open_english_tasked_datasets
          if "chinese" not in x.id
      ]
  
      sorted_small_open_english_tasked_datasets = sorted(
          [
              (x.downloads, x)
              for x in small_open_english_tasked_datasets
          ],
          key=lambda x: x[0],
          reverse=True,
      )
  
      # NOTES:
      # Run like pytest -s -v create_data.py::test_get_open_datasets &> getdata9.log
      # See what needs config passed and add:
      # grep 'load_dataset(' getdata9.log|grep -v data_id|less -S
      # grep "pip install" getdata9.log
      # NOTE: Some datasets have default config, but others are there.  Don't know how to access them.
  
      """
      https://huggingface.co/datasets/wikihow/blob/main/wikihow.py
      https://github.com/mahnazkoupaee/WikiHow-Dataset
      https://ucsb.box.com/s/ap23l8gafpezf4tq3wapr6u8241zz358
      https://ucsb.app.box.com/s/ap23l8gafpezf4tq3wapr6u8241zz358
      """
  
      """
      # some ambiguous or non-commercial datasets
      https://github.com/PhoebusSi/alpaca-CoT
      """
  
      timeout = 3 * 60
      # laion/OIG takes longer
      for (
          num_downloads,
          dataset,
      ) in sorted_small_open_english_tasked_datasets:
          data_id = dataset.id
          func = do_one
          args = (data_id, num_downloads)
          kwargs = {}
          with ProcessPoolExecutor(max_workers=1) as executor:
              future = executor.submit(func, *args, **kwargs)
              try:
                  future.result(timeout=timeout)
              except concurrent.futures.TimeoutError:
                  print(
                      "\n\ndata_id %s timeout\n\n" % data_id, flush=True
                  )
              for child in psutil.Process(os.getpid()).children(
                  recursive=True
              ):
                  os.kill(child.pid, signal.SIGINT)
                  os.kill(child.pid, signal.SIGTERM)
                  os.kill(child.pid, signal.SIGKILL)
  
imports: import ast
  import concurrent.futures
  import contextlib
  import hashlib
  import json
  import os
  import shutil
  import signal
  import sys
  import traceback
  from concurrent.futures import ProcessPoolExecutor
  import psutil
  import pytest
  import pandas
  import numpy
  from tqdm import tqdm
  from utils import flatten_list, remove
  from joblib import Parallel, delayed, effective_n_jobs
  from sklearn.utils import gen_even_slices
  from sklearn.utils.validation import _num_samples
label_desc: Use image node_id 20 for calling a global function with example usage: test_get_open_datasets() without return types