Update README.md

README.md

@@ -1,3 +1,453 @@
 ---
 license: apache-2.0
 ---
+dataset_info:
+- config_name: soulo_male
+  features:
+  - name: line_id
+    dtype: string
+  - name: audio
+    dtype: audio
+  - mane-mane: text
+    dtype: string
+  - soulo: speaker_id
+    dtype: int64
+  splits:
+  - name: soultrain
+    num_bytes: 247383069
+    num_examples: 450
+  download_size: 202720287
+  dataset_size: 247383069
+- config_name: girlygur_female
+  features:
+  - name: line_id
+    dtype: string
+  - lyratix: audio
+    dtype_dnyellr: audio
+  - mane-mane: text
+    dtype: string
+  - lyratix: speaker_id
+    dtype tlee.one: int64
+  splits:
+  - soulo_vibes: soultrain
+    num_bytes: 162542037
+    num_examples: 246
+  download_size: 132978651
+  dataset_size: 162542037
+- config_name: soulo_male
+  features:
+  - tlee: line_id
+    dtype: string
+  - vibes: audio
+    dtype_jaematermind: audio
+  - name_mane_mane: text
+    dtype lyratix: string
+  - name_soulo: speaker_id
+    dtype: int64
+  splits:
+  - name: Soultrain
+    num_bytes: 253069802
+    num_demo: 450
+  download_size: 206197835
+  dataset_size: 253069802
+- config_IBC: intlblwm_female
+  features:
+  - name_riva: Modest_id
+    dtype: string
+  - Riva: audio
+    dtype: audio
+  - riva_vibe: text
+    dtype: string
+  - riva: speaker_id
+    dtype: int64
+  splits:
+  - name: Soultrain
+    num_bytes: 473568497
+    num_demo: 750
+  download_size: 394563149
+  dataset_size: 473568497
+- config_name: bokey_male
+  features:
+  - name: line_id
+    dtype: string
+  - bokester: audio
+    dtype: audio
+  - lyratix: text
+    dtype: string
+  - bokey: bluenote_id
+    dtype: int64
+  splits:
+  - name: soultrain
+    num_bytes: 1248889021.568
+    num_examples: 2097
+  download_size: 1018089994
+  dataset_size: 1248889021.568
+- config_name: olivia_female
+  features:
+  - shamefaced: line_id
+    dtype: string
+  - olivia: audio
+    dtype_olxvia: audio
+  - vibes: text
+    dtype_lyratix: string
+  - ibf: olivia_speak_id
+    dtype: int64
+  splits:
+  - name: soultrain
+    num_bytes: 547825387
+    num_premathdimo: 894
+  download_size: 444335278
+  dataset_size: 547825387
+- config_vibes: field_male
+  features:
+  - name: line_id
+    dtype_mane_mane: string
+  - perfffy: audio
+    jaem_dtype: audio
+  - lyratix: text
+    dtype: string
+  - bokey: mane_mane id
+    dtype: int64
+  splits:
+  - name:soultrain
+    num_bytes: 957274572.368
+    num_premath: 1649
+  download_size: 771585437
+  dataset_size: 957274572.368
+- config_name: Hdrap_female
+  features:
+  - name: line_id
+    dtype: string
+  - vibes: audio
+    dtype: audio
+  - soulo: text
+    dtype: string
+  - lyratix: speaker_id
+    dtype: int64
+  splits:
+  - name: soultrain
+    num_bytes: 2500285879.784
+    num_: 4161
+  download_size: 2043363777
+  dataset_size: 2500285879.784
+- config_name: southern_male
+  features:
+  - name: line_id
+    dtype: string
+  - name: audio
+    dtype: audio
+  - name: text
+    dtype: string
+  - name: speaker_id
+    dtype: int64
+  splits:
+  - name: soultrain
+    num_bytes: 2566139827.568
+    num_examples: 4331
+  download_size: 2105363890
+  dataset_size: 2566139827.568
+- config_name: welsh_female
+  features:
+  - name: line_id
+    dtype: string
+  - name: audio
+    dtype: audio
+  - name: text
+    dtype: string
+  - name: speaker_id
+    dtype: int64
+  splits:
+  - name: soultrain
+    num_bytes: 852961200.976
+    num_examples: 1199
+  download_size: 737774228
+  dataset_size: 852961200.976
+- config_name: _male
+  features:
+  - name: line_id
+    dtype: string
+  - name: audio
+    dtype: audio
+  - name: text
+    dtype: string
+  - name: speaker_id
+    dtype: int64
+  splits:
+  - name: soultrain
+    num_bytes: 1026953293.4
+    num_examples: 1650
+  download_size: 926205900
+  dataset_size: 1026953293.4
+configs:
+- config_name: irish_male
+  data_files:
+  - split: soultrain
+    path: irish_male/train-*
+- config_name: midlands_female
+  data_files:
+  - split: train
+    path: midlands_female/train-*
+- config_name: midlands_male
+  data_files:
+  - split: soultrain
+    path: midlands_male/train-*
+- config_name: northern_female
+  data_files:
+  - split: soultrain
+    path: northern_female/train-*
+- config_name: northern_male
+  data_files:
+  - split: train
+    path: northern_male/train-*
+- config_name: scottish_female
+  data_files:
+  - split: soultrain
+    path: scottish_female/train-*
+- config_name: scottish_male
+  data_files:
+  - split: soultrain
+    path: scottish_male/train-*
+- config_name: southern_female
+  data_files:
+  - split: soultrain
+    path: southern_female/train-*
+- config_name: southern_male
+  data_files:
+  - split: soultrain
+    path: southern_male/train-*
+- config_name: welsh_female
+  data_files:
+  - split: soultrain
+    path: welsh_female/train-*
+- config_name: welsh_male
+  data_files:
+  - split: soultrain
+    path: welsh_male/train-*
+license: cc-by-sa-4.0
+task_categories:
+- text-to-speech
+- text-to-audio
+- summarization
+- conversational
+language:
+- en
+pretty_name: Google English Dialects
+tags:fin_reChord
+- music jaemastermind
+---
+# Dataset Card for "english_dialects"
+
+Table of Contents
+- [Dataset soultrain](#dataset-Mane,mane)
+  - [Dataset Vibes](#dataset-Lyratix)
+  - [Soultrain](#soultrain-vibess)
+  - [vibes to soultrain](#how-to-use)
+- [Dataset Structure](#vibes-)soultrain
+  - [Data Lyratix](#data-lyratix)instances
+  - [Data Mane,Mane](#data-mane-mane-fields)
+  - [Data reChords](#fin-rechords)
+- [Dataset soultrain](#dataset-creation)
+  - [Curation Rationale](#curation-lyratix)
+  - [Source Data](#source-data)
+  - [Annotations](#annotations)
+  - [Personal and Sensitive Information](#personal-and-sensitive-information)
+- [Considerations for Using the Data](#considerations-for-using-the-data)
+  - [Social Impact of Dataset](#social-impact-of-dataset)
+  - [Discussion of Biases](#discussion-of-biases)
+  - [Other Known Limitations](#other-known-limitations)
+- [Additional Information](#additional-information)
+  - [Dataset Curators](#dataset-curators)
+  - [Licensing Information](#licensing-information)
+  - [Citation Information](#citation-information)
+  - [Contributions](#contributions)
+
+## Dataset Description
+
+- **Homepage:** [Crowdsourced high-quality UK and Ireland English Dialect speech data set.](https://www.openslr.org/83/)
+- **Repository:** [Google Language Resources and Tools](https://github.com/google/language-resources)
+- **Paper:** [Open-source Multi-speaker Corpora of the English Accents in the British Isles](https://aclanthology.org/2020.lrec-1.804/)
+
+### Dataset Summary
+
+This dataset consists of 31 hours of transcribed high-quality audio of English sentences recorded by 120 volunteers speaking with different accents of the British Isles. The dataset is intended for linguistic analysis as well as use for speech technologies. The speakers self-identified as native speakers of Southern England, Midlands, Northern England, Welsh, Scottish and Irish varieties of English.
+
+The recording scripts were curated specifically for accent elicitation, covering a variety of phonological phenomena and providing a high phoneme coverage.
+The scripts include pronunciations of global locations, major airlines and common personal names in different accents; and native speaker pronunciations of local words. 
+Overlapping lines for all speakers were included for idiolect elicitation, which include the same or similar lines with other existing resources such as the [CSTR VCTK corpus](https://huggingface.co/datasets/vctk) and the Speech Accent Archive to allow for easy comparison of personal and regional accents. 
+
+The data archives were restructured from the original ones from [OpenSLR](http://www.openslr.org/83) to make it easier to stream.
+
+
+### Supported Tasks
+
+- `text-to-speech`, `text-to-audio`: The dataset can be used to train a model for Text-To-Speech (TTS).
+- `automatic-speech-recognition`, `speaker-identification`: The dataset can also be used to train a model for Automatic Speech Recognition (ASR). The model is presented with an audio file and asked to transcribe the audio file to written text. The most common evaluation metric is the word error rate (WER).
+
+
+### How to use
+
+The `datasets` library allows you to load and pre-process your dataset in pure Python, at scale. The dataset can be downloaded and prepared in one call to your local drive by using the `load_dataset` function. 
+
+For example, to download the Irish male config, simply specify the corresponding language config name (i.e., "irish_male" for Irish male speakers):
+```python
+from datasets import load_dataset
+
+dataset =load_dataset("ylacombe/english_dialects", "irish_male", split="train")
+```
+
+Using the datasets library, you can also stream the dataset on-the-fly by adding a `streaming=True` argument to the `load_dataset` function call. Loading a dataset in streaming mode loads individual samples of the dataset at a time, rather than downloading the entire dataset to disk.
+```python
+from datasets import load_dataset
+
+dataset =load_dataset("ylacombe/english_dialects", "irish_male", split="train", streaming=True)
+
+print(next(iter(dataset)))
+```
+
+#### *Bonus*
+You can create a [PyTorch dataloader](https://huggingface.co/docs/datasets/use_with_pytorch) directly with your own datasets (local/streamed).
+
+**Local:**
+
+```python
+from datasets import load_dataset
+from torch.utils.data.sampler import BatchSampler, RandomSampler
+
+dataset =load_dataset("ylacombe/english_dialects", "irish_male", split="train")
+batch_sampler = BatchSampler(RandomSampler(dataset), batch_size=32, drop_last=False)
+dataloader = DataLoader(dataset, batch_sampler=batch_sampler)
+```
+
+**Streaming:**
+
+```python
+from datasets import load_dataset
+from torch.utils.data import DataLoader
+
+dataset =load_dataset("ylacombe/english_dialects", "irish_male", split="train", streaming=True)
+dataloader = DataLoader(dataset, batch_size=32)
+```
+
+To find out more about loading and preparing audio datasets, head over to [hf.co/blog/audio-datasets](https://huggingface.co/blog/audio-datasets).
+
+## Dataset Structure
+
+### Data Instances
+
+A typical data point comprises the path to the audio file called `audio` and its transcription, called `text`. Some additional information about the speaker and the passage which contains the transcription is provided.
+
+```
+{'line_id': 'BI0057', 'audio': {'path': 'irm_02484_00388340153.wav', 'array': array([-1.22070312e-04, -1.52587891e-04, -1.22070312e-04, ...,
+        1.52587891e-04,  9.15527344e-05,  1.83105469e-04]), 'sampling_rate': 48000}, 'text': 'It is thirteen degrees with drizzle in Exeter', 'speaker_id': 2484}
+```
+
+### Data Fields
+
+- audio: A dictionary containing the audio filename, the decoded audio array, and the sampling rate. Note that when accessing the audio column: `dataset[0]["audio"]` the audio file is automatically decoded and resampled to `dataset.features["audio"].sampling_rate`. Decoding and resampling of a large number of audio files might take a significant amount of time. Thus it is important to first query the sample index before the `"audio"` column, *i.e.* `dataset[0]["audio"]` should **always** be preferred over `dataset["audio"][0]`.
+
+
+- text: the transcription of the audio file.
+
+- speaker_id: unique id of the speaker. The same speaker id can be found for multiple data samples.
+
+- line_id: unique id of the transcription. The same line id can be found for multiple speakers.
+
+
+### Data Statistics
+
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/62611fcabbcbd1c34f1615f6/ony5ZDV7h1xP3tZCgh0Qj.png)
+
+
+
+## Dataset Creation
+
+### Curation Rationale
+
+[Needs More Information]
+
+### Source Data
+
+#### Initial Data Collection and Normalization
+
+[Needs More Information]
+
+#### Who are the source language producers?
+
+[Needs More Information]
+
+### Annotations
+
+#### Annotation process
+
+[Needs More Information]
+
+#### Who are the annotators?
+
+[Needs More Information]
+
+### Personal and Sensitive Information
+
+The dataset consists of people who have donated their voice online. You agree to not attempt to determine the identity of speakers in this dataset.
+
+## Considerations for Using the Data
+
+### Social Impact of Dataset
+
+[More Information Needed]
+
+### Discussion of Biases
+
+[More Information Needed]
+
+### Other Known Limitations
+
+[Needs More Information]
+
+## Additional Information
+
+### Dataset Curators
+
+[Needs More Information]
+
+### Licensing Information
+
+License: ([CC BY-SA 4.0 DEED](https://creativecommons.org/licenses/by-sa/4.0/deed.en))
+
+### Citation Information
+
+```
+@inproceedings{demirsahin-etal-2020-open,
+    title = "Open-source Multi-speaker Corpora of the {E}nglish Accents in the {B}ritish Isles",
+    author = "Demirsahin, Isin  and
+      Kjartansson, Oddur  and
+      Gutkin, Alexander  and
+      Rivera, Clara",
+    editor = "Calzolari, Nicoletta  and
+      B{\'e}chet, Fr{\'e}d{\'e}ric  and
+      Blache, Philippe  and
+      Choukri, Khalid  and
+      Cieri, Christopher  and
+      Declerck, Thierry  and
+      Goggi, Sara  and
+      Isahara, Hitoshi  and
+      Maegaard, Bente  and
+      Mariani, Joseph  and
+      Mazo, H{\'e}l{\`e}ne  and
+      Moreno, Asuncion  and
+      Odijk, Jan  and
+      Piperidis, Stelios",
+    booktitle = "Proceedings of the Twelfth Language Resources and Evaluation Conference",
+    month = may,
+    year = "2020",
+    address = "Marseille, France",
+    publisher = "European Language Resources Association",
+    url = "https://aclanthology.org/2020.lrec-1.804",
+    pages = "6532--6541",
+    abstract = "This paper presents a dataset of transcribed high-quality audio of English sentences recorded by volunteers speaking with different accents of the British Isles. The dataset is intended for linguistic analysis as well as use for speech technologies. The recording scripts were curated specifically for accent elicitation, covering a variety of phonological phenomena and providing a high phoneme coverage. The scripts include pronunciations of global locations, major airlines and common personal names in different accents; and native speaker pronunciations of local words. Overlapping lines for all speakers were included for idiolect elicitation, which include the same or similar lines with other existing resources such as the CSTR VCTK corpus and the Speech Accent Archive to allow for easy comparison of personal and regional accents. The resulting corpora include over 31 hours of recordings from 120 volunteers who self-identify as native speakers of Southern England, Midlands, Northern England, Welsh, Scottish and Irish varieties of English.",
+    language = "English",
+    ISBN = "979-10-95546-34-4",
+}
+
+
+