prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Whisper workflow supervision `end` may be incorrect
  So I ran the model on one of the AMI headset recordings (~5000s) and it seems like the "end" actually shows the end of the segment, not the duration. Here is the JSON containing the `results["segments"]`: https://drive.google.com/file/d/169igkcDY2SmMs5k3hOhHip89T4MQDnKs/view?usp=sharing
  
  _Originally posted by @desh2608 in https://github.com/lhotse-speech/lhotse/pull/834#discussion_r988376898_
        
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `lhotse/workflows/whisper.py`
  Content:
  ```
  1 import logging
  2 from typing import Any, Generator, List, Optional, Union
  3 
  4 import torch
  5 
  6 from lhotse import CutSet, MonoCut, Recording, RecordingSet, SupervisionSegment
  7 from lhotse.qa import trim_supervisions_to_recordings
  8 from lhotse.utils import fastcopy, is_module_available
  9 
  10 
  11 def annotate_with_whisper(
  12     manifest: Union[RecordingSet, CutSet],
  13     language: Optional[str] = None,
  14     model_name: str = "base",
  15     device: str = "cpu",
  16 ) -> Generator[MonoCut, None, None]:
  17     """
  18     Use OpenAI Whisper model to annotate either RECORDINGS_MANIFEST, RECORDINGS_DIR, or CUTS_MANIFEST.
  19     It will perform automatic segmentation, transcription, and language identification. If
  20     the first argument is a CutSet, it will overwrite the supervisions with the results of the inference.
  21 
  22     Note: this is an experimental feature of Lhotse, and is not guaranteed to yield
  23     high quality of data.
  24 
  25     See the original repo for more details: https://github.com/openai/whisper
  26 
  27     :param manifest: a ``RecordingSet`` or ``CutSet`` object.
  28     :param language: specify the language if known upfront, otherwise it will be auto-detected.
  29     :param model_name: one of available Whisper variants (base, medium, large, etc.).
  30     :param device: Where to run the inference (cpu, cuda, etc.).
  31     :return: a generator of cuts (use ``CutSet.open_writer()`` to write them).
  32     """
  33     assert is_module_available("whisper"), (
  34         "This function expects OpenAI Whisper to be installed. "
  35         "You can install it via 'pip install git+https://github.com/openai/whisper.git' "
  36         "(see https://github.com/openai/whisper for details)."
  37     )
  38 
  39     if isinstance(manifest, RecordingSet):
  40         yield from _annotate_recordings(manifest, language, model_name, device)
  41     elif isinstance(manifest, CutSet):
  42         yield from _annotate_cuts(manifest, language, model_name, device)
  43     else:
  44         raise ValueError("The ``manifest`` must be either a RecordingSet or a CutSet.")
  45 
  46 
  47 def _annotate_recordings(
  48     recordings: RecordingSet, language: str, model_name: str, device: str
  49 ):
  50     """
  51     Helper function that annotates a RecordingSet with Whisper.
  52     """
  53     import whisper
  54 
  55     model = whisper.load_model(model_name, device=device)
  56 
  57     for recording in recordings:
  58         if recording.num_channels > 1:
  59             logging.warning(
  60                 f"Skipping recording '{recording.id}'. It has {recording.num_channels} channels, "
  61                 f"but we currently only support mono input."
  62             )
  63             continue
  64         audio = torch.from_numpy(recording.resample(16000).load_audio()).squeeze(0)
  65         result = whisper.transcribe(model=model, audio=audio, language=language)
  66         supervisions = [
  67             SupervisionSegment(
  68                 id=f"{recording.id}-{segment['id']:06d}",
  69                 recording_id=recording.id,
  70                 start=round(segment["start"], ndigits=8),
  71                 duration=round(segment["end"], ndigits=8),
  72                 text=segment["text"].strip(),
  73                 language=result["language"],
  74             )
  75             for segment in result["segments"]
  76         ]
  77         cut = recording.to_cut()
  78         if supervisions:
  79             supervisions = _postprocess_timestamps(supervisions)
  80             cut.supervisions = list(
  81                 trim_supervisions_to_recordings(
  82                     recordings=recording, supervisions=supervisions, verbose=False
  83                 )
  84             )
  85         yield cut
  86 
  87 
  88 def _annotate_cuts(cuts: CutSet, language: str, model_name: str, device: str):
  89     """
  90     Helper function that annotates a CutSet with Whisper.
  91     """
  92     import whisper
  93 
  94     model = whisper.load_model(model_name, device=device)
  95 
  96     for cut in cuts:
  97         if cut.num_channels > 1:
  98             logging.warning(
  99                 f"Skipping cut '{cut.id}'. It has {cut.num_channels} channels, "
  100                 f"but we currently only support mono input."
  101             )
  102             continue
  103         audio = torch.from_numpy(cut.resample(16000).load_audio()).squeeze(0)
  104         result = whisper.transcribe(model=model, audio=audio, language=language)
  105         supervisions = [
  106             SupervisionSegment(
  107                 id=f"{cut.id}-{segment['id']:06d}",
  108                 recording_id=cut.recording_id,
  109                 start=round(segment["start"], ndigits=8),
  110                 duration=max(cut.duration, round(segment["end"], ndigits=8)),
  111                 text=segment["text"].strip(),
  112                 language=result["language"],
  113             )
  114             for segment in result["segments"]
  115         ]
  116         new_cut = fastcopy(cut, supervisions=_postprocess_timestamps(supervisions))
  117         yield new_cut
  118 
  119 
  120 def _postprocess_timestamps(supervisions: List[SupervisionSegment]):
  121     """
  122     Whisper tends to have a lot of overlapping segments due to inaccurate end timestamps.
  123     Under a strong assumption that the input speech is non-overlapping, we can fix that
  124     by always truncating to the start timestamp of the next segment.
  125     """
  126     from cytoolz import sliding_window
  127 
  128     if len(supervisions) < 2:
  129         return supervisions
  130     out = []
  131     for cur, nxt in sliding_window(2, supervisions):
  132         if cur.end > nxt.start:
  133             cur = cur.trim(end=nxt.start)
  134         out.append(cur)
  135     out.append(nxt)
  136     return out
  137
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/lhotse/workflows/whisper.py b/lhotse/workflows/whisper.py
  --- a/lhotse/workflows/whisper.py
  +++ b/lhotse/workflows/whisper.py
  @@ -3,7 +3,14 @@
   
   import torch
   
  -from lhotse import CutSet, MonoCut, Recording, RecordingSet, SupervisionSegment
  +from lhotse import (
  +    CutSet,
  +    MonoCut,
  +    Recording,
  +    RecordingSet,
  +    SupervisionSegment,
  +    add_durations,
  +)
   from lhotse.qa import trim_supervisions_to_recordings
   from lhotse.utils import fastcopy, is_module_available
   
  @@ -68,7 +75,9 @@
                   id=f"{recording.id}-{segment['id']:06d}",
                   recording_id=recording.id,
                   start=round(segment["start"], ndigits=8),
  -                duration=round(segment["end"], ndigits=8),
  +                duration=add_durations(
  +                    segment["end"], -segment["start"], sampling_rate=16000
  +                ),
                   text=segment["text"].strip(),
                   language=result["language"],
               )
  @@ -107,7 +116,12 @@
                   id=f"{cut.id}-{segment['id']:06d}",
                   recording_id=cut.recording_id,
                   start=round(segment["start"], ndigits=8),
  -                duration=max(cut.duration, round(segment["end"], ndigits=8)),
  +                duration=max(
  +                    cut.duration,
  +                    add_durations(
  +                        segment["end"], -segment["start"], sampling_rate=16000
  +                    ),
  +                ),
                   text=segment["text"].strip(),
                   language=result["language"],
               )
  
verification_info: {"golden_diff": "diff --git a/lhotse/workflows/whisper.py b/lhotse/workflows/whisper.py\n--- a/lhotse/workflows/whisper.py\n+++ b/lhotse/workflows/whisper.py\n@@ -3,7 +3,14 @@\n \n import torch\n \n-from lhotse import CutSet, MonoCut, Recording, RecordingSet, SupervisionSegment\n+from lhotse import (\n+    CutSet,\n+    MonoCut,\n+    Recording,\n+    RecordingSet,\n+    SupervisionSegment,\n+    add_durations,\n+)\n from lhotse.qa import trim_supervisions_to_recordings\n from lhotse.utils import fastcopy, is_module_available\n \n@@ -68,7 +75,9 @@\n                 id=f\"{recording.id}-{segment['id']:06d}\",\n                 recording_id=recording.id,\n                 start=round(segment[\"start\"], ndigits=8),\n-                duration=round(segment[\"end\"], ndigits=8),\n+                duration=add_durations(\n+                    segment[\"end\"], -segment[\"start\"], sampling_rate=16000\n+                ),\n                 text=segment[\"text\"].strip(),\n                 language=result[\"language\"],\n             )\n@@ -107,7 +116,12 @@\n                 id=f\"{cut.id}-{segment['id']:06d}\",\n                 recording_id=cut.recording_id,\n                 start=round(segment[\"start\"], ndigits=8),\n-                duration=max(cut.duration, round(segment[\"end\"], ndigits=8)),\n+                duration=max(\n+                    cut.duration,\n+                    add_durations(\n+                        segment[\"end\"], -segment[\"start\"], sampling_rate=16000\n+                    ),\n+                ),\n                 text=segment[\"text\"].strip(),\n                 language=result[\"language\"],\n             )\n", "issue": "Whisper workflow supervision `end` may be incorrect\nSo I ran the model on one of the AMI headset recordings (~5000s) and it seems like the \"end\" actually shows the end of the segment, not the duration. Here is the JSON containing the `results[\"segments\"]`: https://drive.google.com/file/d/169igkcDY2SmMs5k3hOhHip89T4MQDnKs/view?usp=sharing\r\n\r\n_Originally posted by @desh2608 in https://github.com/lhotse-speech/lhotse/pull/834#discussion_r988376898_\r\n      \n", "before_files": [{"content": "import logging\nfrom typing import Any, Generator, List, Optional, Union\n\nimport torch\n\nfrom lhotse import CutSet, MonoCut, Recording, RecordingSet, SupervisionSegment\nfrom lhotse.qa import trim_supervisions_to_recordings\nfrom lhotse.utils import fastcopy, is_module_available\n\n\ndef annotate_with_whisper(\n    manifest: Union[RecordingSet, CutSet],\n    language: Optional[str] = None,\n    model_name: str = \"base\",\n    device: str = \"cpu\",\n) -> Generator[MonoCut, None, None]:\n    \"\"\"\n    Use OpenAI Whisper model to annotate either RECORDINGS_MANIFEST, RECORDINGS_DIR, or CUTS_MANIFEST.\n    It will perform automatic segmentation, transcription, and language identification. If\n    the first argument is a CutSet, it will overwrite the supervisions with the results of the inference.\n\n    Note: this is an experimental feature of Lhotse, and is not guaranteed to yield\n    high quality of data.\n\n    See the original repo for more details: https://github.com/openai/whisper\n\n    :param manifest: a ``RecordingSet`` or ``CutSet`` object.\n    :param language: specify the language if known upfront, otherwise it will be auto-detected.\n    :param model_name: one of available Whisper variants (base, medium, large, etc.).\n    :param device: Where to run the inference (cpu, cuda, etc.).\n    :return: a generator of cuts (use ``CutSet.open_writer()`` to write them).\n    \"\"\"\n    assert is_module_available(\"whisper\"), (\n        \"This function expects OpenAI Whisper to be installed. \"\n        \"You can install it via 'pip install git+https://github.com/openai/whisper.git' \"\n        \"(see https://github.com/openai/whisper for details).\"\n    )\n\n    if isinstance(manifest, RecordingSet):\n        yield from _annotate_recordings(manifest, language, model_name, device)\n    elif isinstance(manifest, CutSet):\n        yield from _annotate_cuts(manifest, language, model_name, device)\n    else:\n        raise ValueError(\"The ``manifest`` must be either a RecordingSet or a CutSet.\")\n\n\ndef _annotate_recordings(\n    recordings: RecordingSet, language: str, model_name: str, device: str\n):\n    \"\"\"\n    Helper function that annotates a RecordingSet with Whisper.\n    \"\"\"\n    import whisper\n\n    model = whisper.load_model(model_name, device=device)\n\n    for recording in recordings:\n        if recording.num_channels > 1:\n            logging.warning(\n                f\"Skipping recording '{recording.id}'. It has {recording.num_channels} channels, \"\n                f\"but we currently only support mono input.\"\n            )\n            continue\n        audio = torch.from_numpy(recording.resample(16000).load_audio()).squeeze(0)\n        result = whisper.transcribe(model=model, audio=audio, language=language)\n        supervisions = [\n            SupervisionSegment(\n                id=f\"{recording.id}-{segment['id']:06d}\",\n                recording_id=recording.id,\n                start=round(segment[\"start\"], ndigits=8),\n                duration=round(segment[\"end\"], ndigits=8),\n                text=segment[\"text\"].strip(),\n                language=result[\"language\"],\n            )\n            for segment in result[\"segments\"]\n        ]\n        cut = recording.to_cut()\n        if supervisions:\n            supervisions = _postprocess_timestamps(supervisions)\n            cut.supervisions = list(\n                trim_supervisions_to_recordings(\n                    recordings=recording, supervisions=supervisions, verbose=False\n                )\n            )\n        yield cut\n\n\ndef _annotate_cuts(cuts: CutSet, language: str, model_name: str, device: str):\n    \"\"\"\n    Helper function that annotates a CutSet with Whisper.\n    \"\"\"\n    import whisper\n\n    model = whisper.load_model(model_name, device=device)\n\n    for cut in cuts:\n        if cut.num_channels > 1:\n            logging.warning(\n                f\"Skipping cut '{cut.id}'. It has {cut.num_channels} channels, \"\n                f\"but we currently only support mono input.\"\n            )\n            continue\n        audio = torch.from_numpy(cut.resample(16000).load_audio()).squeeze(0)\n        result = whisper.transcribe(model=model, audio=audio, language=language)\n        supervisions = [\n            SupervisionSegment(\n                id=f\"{cut.id}-{segment['id']:06d}\",\n                recording_id=cut.recording_id,\n                start=round(segment[\"start\"], ndigits=8),\n                duration=max(cut.duration, round(segment[\"end\"], ndigits=8)),\n                text=segment[\"text\"].strip(),\n                language=result[\"language\"],\n            )\n            for segment in result[\"segments\"]\n        ]\n        new_cut = fastcopy(cut, supervisions=_postprocess_timestamps(supervisions))\n        yield new_cut\n\n\ndef _postprocess_timestamps(supervisions: List[SupervisionSegment]):\n    \"\"\"\n    Whisper tends to have a lot of overlapping segments due to inaccurate end timestamps.\n    Under a strong assumption that the input speech is non-overlapping, we can fix that\n    by always truncating to the start timestamp of the next segment.\n    \"\"\"\n    from cytoolz import sliding_window\n\n    if len(supervisions) < 2:\n        return supervisions\n    out = []\n    for cur, nxt in sliding_window(2, supervisions):\n        if cur.end > nxt.start:\n            cur = cur.trim(end=nxt.start)\n        out.append(cur)\n    out.append(nxt)\n    return out\n", "path": "lhotse/workflows/whisper.py"}], "after_files": [{"content": "import logging\nfrom typing import Any, Generator, List, Optional, Union\n\nimport torch\n\nfrom lhotse import (\n    CutSet,\n    MonoCut,\n    Recording,\n    RecordingSet,\n    SupervisionSegment,\n    add_durations,\n)\nfrom lhotse.qa import trim_supervisions_to_recordings\nfrom lhotse.utils import fastcopy, is_module_available\n\n\ndef annotate_with_whisper(\n    manifest: Union[RecordingSet, CutSet],\n    language: Optional[str] = None,\n    model_name: str = \"base\",\n    device: str = \"cpu\",\n) -> Generator[MonoCut, None, None]:\n    \"\"\"\n    Use OpenAI Whisper model to annotate either RECORDINGS_MANIFEST, RECORDINGS_DIR, or CUTS_MANIFEST.\n    It will perform automatic segmentation, transcription, and language identification. If\n    the first argument is a CutSet, it will overwrite the supervisions with the results of the inference.\n\n    Note: this is an experimental feature of Lhotse, and is not guaranteed to yield\n    high quality of data.\n\n    See the original repo for more details: https://github.com/openai/whisper\n\n    :param manifest: a ``RecordingSet`` or ``CutSet`` object.\n    :param language: specify the language if known upfront, otherwise it will be auto-detected.\n    :param model_name: one of available Whisper variants (base, medium, large, etc.).\n    :param device: Where to run the inference (cpu, cuda, etc.).\n    :return: a generator of cuts (use ``CutSet.open_writer()`` to write them).\n    \"\"\"\n    assert is_module_available(\"whisper\"), (\n        \"This function expects OpenAI Whisper to be installed. \"\n        \"You can install it via 'pip install git+https://github.com/openai/whisper.git' \"\n        \"(see https://github.com/openai/whisper for details).\"\n    )\n\n    if isinstance(manifest, RecordingSet):\n        yield from _annotate_recordings(manifest, language, model_name, device)\n    elif isinstance(manifest, CutSet):\n        yield from _annotate_cuts(manifest, language, model_name, device)\n    else:\n        raise ValueError(\"The ``manifest`` must be either a RecordingSet or a CutSet.\")\n\n\ndef _annotate_recordings(\n    recordings: RecordingSet, language: str, model_name: str, device: str\n):\n    \"\"\"\n    Helper function that annotates a RecordingSet with Whisper.\n    \"\"\"\n    import whisper\n\n    model = whisper.load_model(model_name, device=device)\n\n    for recording in recordings:\n        if recording.num_channels > 1:\n            logging.warning(\n                f\"Skipping recording '{recording.id}'. It has {recording.num_channels} channels, \"\n                f\"but we currently only support mono input.\"\n            )\n            continue\n        audio = torch.from_numpy(recording.resample(16000).load_audio()).squeeze(0)\n        result = whisper.transcribe(model=model, audio=audio, language=language)\n        supervisions = [\n            SupervisionSegment(\n                id=f\"{recording.id}-{segment['id']:06d}\",\n                recording_id=recording.id,\n                start=round(segment[\"start\"], ndigits=8),\n                duration=add_durations(\n                    segment[\"end\"], -segment[\"start\"], sampling_rate=16000\n                ),\n                text=segment[\"text\"].strip(),\n                language=result[\"language\"],\n            )\n            for segment in result[\"segments\"]\n        ]\n        cut = recording.to_cut()\n        if supervisions:\n            supervisions = _postprocess_timestamps(supervisions)\n            cut.supervisions = list(\n                trim_supervisions_to_recordings(\n                    recordings=recording, supervisions=supervisions, verbose=False\n                )\n            )\n        yield cut\n\n\ndef _annotate_cuts(cuts: CutSet, language: str, model_name: str, device: str):\n    \"\"\"\n    Helper function that annotates a CutSet with Whisper.\n    \"\"\"\n    import whisper\n\n    model = whisper.load_model(model_name, device=device)\n\n    for cut in cuts:\n        if cut.num_channels > 1:\n            logging.warning(\n                f\"Skipping cut '{cut.id}'. It has {cut.num_channels} channels, \"\n                f\"but we currently only support mono input.\"\n            )\n            continue\n        audio = torch.from_numpy(cut.resample(16000).load_audio()).squeeze(0)\n        result = whisper.transcribe(model=model, audio=audio, language=language)\n        supervisions = [\n            SupervisionSegment(\n                id=f\"{cut.id}-{segment['id']:06d}\",\n                recording_id=cut.recording_id,\n                start=round(segment[\"start\"], ndigits=8),\n                duration=max(\n                    cut.duration,\n                    add_durations(\n                        segment[\"end\"], -segment[\"start\"], sampling_rate=16000\n                    ),\n                ),\n                text=segment[\"text\"].strip(),\n                language=result[\"language\"],\n            )\n            for segment in result[\"segments\"]\n        ]\n        new_cut = fastcopy(cut, supervisions=_postprocess_timestamps(supervisions))\n        yield new_cut\n\n\ndef _postprocess_timestamps(supervisions: List[SupervisionSegment]):\n    \"\"\"\n    Whisper tends to have a lot of overlapping segments due to inaccurate end timestamps.\n    Under a strong assumption that the input speech is non-overlapping, we can fix that\n    by always truncating to the start timestamp of the next segment.\n    \"\"\"\n    from cytoolz import sliding_window\n\n    if len(supervisions) < 2:\n        return supervisions\n    out = []\n    for cur, nxt in sliding_window(2, supervisions):\n        if cur.end > nxt.start:\n            cur = cur.trim(end=nxt.start)\n        out.append(cur)\n    out.append(nxt)\n    return out\n", "path": "lhotse/workflows/whisper.py"}]}