prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Links to BioViL paper need to be updated
  We are still using the arXiv preprint. But this should be solved first:
  - https://github.com/microsoft/hi-ml/pull/730#issuecomment-1419298653
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py`
  Content:
  ```
  1 #  ------------------------------------------------------------------------------------------
  2 #  Copyright (c) Microsoft Corporation. All rights reserved.
  3 #  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
  4 #  ------------------------------------------------------------------------------------------
  5 
  6 from typing import Any, Optional, Tuple, Union
  7 
  8 import torch
  9 import torch.nn.functional as F
  10 from torch import nn
  11 from torch import Tensor as T
  12 from transformers import BertForMaskedLM
  13 from transformers.modeling_outputs import ModelOutput
  14 
  15 from health_multimodal.text.model.configuration_cxrbert import CXRBertConfig
  16 
  17 BERTTupleOutput = Tuple[T, T, T, T, T]
  18 
  19 
  20 class CXRBertOutput(ModelOutput):
  21     last_hidden_state: torch.FloatTensor
  22     logits: torch.FloatTensor
  23     cls_projected_embedding: Optional[torch.FloatTensor] = None
  24     hidden_states: Optional[Tuple[torch.FloatTensor]] = None
  25     attentions: Optional[Tuple[torch.FloatTensor]] = None
  26 
  27 
  28 class BertProjectionHead(nn.Module):
  29     """Projection head to be used with BERT CLS token.
  30 
  31     This is similar to ``BertPredictionHeadTransform`` in HuggingFace.
  32 
  33     :param config: Configuration for BERT.
  34     """
  35 
  36     def __init__(self, config: CXRBertConfig) -> None:
  37         super().__init__()
  38         self.dense_to_hidden = nn.Linear(config.hidden_size, config.projection_size)
  39         self.transform_act_fn = nn.functional.gelu
  40         self.LayerNorm = nn.LayerNorm(config.projection_size, eps=1e-12)
  41         self.dense_to_output = nn.Linear(config.projection_size, config.projection_size)
  42 
  43     def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
  44         hidden_states = self.dense_to_hidden(hidden_states)
  45         hidden_states = self.transform_act_fn(hidden_states)
  46         hidden_states = self.LayerNorm(hidden_states)
  47         hidden_states = self.dense_to_output(hidden_states)
  48 
  49         return hidden_states
  50 
  51 
  52 class CXRBertModel(BertForMaskedLM):
  53     """
  54     Implements the CXR-BERT model outlined in the manuscript:
  55     Boecking et al. "Making the Most of Text Semantics to Improve Biomedical Vision-Language Processing", 2022
  56     https://arxiv.org/abs/2204.09817
  57 
  58     Extends the HuggingFace BertForMaskedLM model by adding a separate projection head. The projection "[CLS]" token is
  59     used to align the latent vectors of image and text modalities.
  60     """
  61 
  62     config_class = CXRBertConfig  # type: ignore
  63 
  64     def __init__(self, config: CXRBertConfig):
  65         super().__init__(config)
  66 
  67         self.cls_projection_head = BertProjectionHead(config)
  68         self.init_weights()
  69 
  70     def forward(
  71         self,
  72         input_ids: torch.Tensor,
  73         attention_mask: torch.Tensor,
  74         token_type_ids: Optional[torch.Tensor] = None,
  75         position_ids: Optional[torch.Tensor] = None,
  76         head_mask: Optional[torch.Tensor] = None,
  77         inputs_embeds: Optional[torch.Tensor] = None,
  78         output_attentions: Optional[bool] = None,
  79         output_hidden_states: Optional[bool] = None,
  80         output_cls_projected_embedding: Optional[bool] = None,
  81         return_dict: Optional[bool] = None,
  82         **kwargs: Any
  83     ) -> Union[BERTTupleOutput, CXRBertOutput]:
  84 
  85         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
  86 
  87         bert_for_masked_lm_output = super().forward(input_ids=input_ids,
  88                                                     attention_mask=attention_mask,
  89                                                     token_type_ids=token_type_ids,
  90                                                     position_ids=position_ids,
  91                                                     head_mask=head_mask,
  92                                                     inputs_embeds=inputs_embeds,
  93                                                     output_attentions=output_attentions,
  94                                                     output_hidden_states=True,
  95                                                     return_dict=True)
  96 
  97         last_hidden_state = bert_for_masked_lm_output.hidden_states[-1]
  98         cls_projected_embedding = self.cls_projection_head(
  99             last_hidden_state[:, 0, :]) if output_cls_projected_embedding else None
  100 
  101         if return_dict:
  102             return CXRBertOutput(
  103                 last_hidden_state=last_hidden_state,
  104                 logits=bert_for_masked_lm_output.logits,
  105                 cls_projected_embedding=cls_projected_embedding,
  106                 hidden_states=bert_for_masked_lm_output.hidden_states if output_hidden_states else None,
  107                 attentions=bert_for_masked_lm_output.attentions,
  108             )
  109         else:
  110             return (
  111                 last_hidden_state,
  112                 bert_for_masked_lm_output.logits,
  113                 cls_projected_embedding,
  114                 bert_for_masked_lm_output.hidden_states,
  115                 bert_for_masked_lm_output.attentions,)
  116 
  117     def get_projected_text_embeddings(self,
  118                                       input_ids: torch.Tensor,
  119                                       attention_mask: torch.Tensor,
  120                                       normalize_embeddings: bool = True) -> torch.Tensor:
  121         """
  122         Returns l2-normalised projected cls token embeddings for the given input token ids and attention mask.
  123         The joint latent space is trained using a contrastive objective between image and text data modalities.
  124 
  125         :param input_ids: (batch_size, sequence_length)
  126         :param attention_mask: (batch_size, sequence_length)
  127         :param normalize_embeddings: Whether to l2-normalise the embeddings.
  128         :return: (batch_size, projection_size)
  129         """
  130 
  131         outputs = self.forward(input_ids=input_ids, attention_mask=attention_mask,
  132                                output_cls_projected_embedding=True, return_dict=True)
  133         assert isinstance(outputs, CXRBertOutput)
  134 
  135         cls_projected_embedding = outputs.cls_projected_embedding
  136         assert cls_projected_embedding is not None
  137 
  138         if normalize_embeddings:
  139             return F.normalize(cls_projected_embedding, dim=1)
  140 
  141         return cls_projected_embedding
  142
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py b/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py
  --- a/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py
  +++ b/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py
  @@ -53,7 +53,7 @@
       """
       Implements the CXR-BERT model outlined in the manuscript:
       Boecking et al. "Making the Most of Text Semantics to Improve Biomedical Vision-Language Processing", 2022
  -    https://arxiv.org/abs/2204.09817
  +    https://link.springer.com/chapter/10.1007/978-3-031-20059-5_1
   
       Extends the HuggingFace BertForMaskedLM model by adding a separate projection head. The projection "[CLS]" token is
       used to align the latent vectors of image and text modalities.
  
verification_info: {"golden_diff": "diff --git a/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py b/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py\n--- a/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py\n+++ b/hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py\n@@ -53,7 +53,7 @@\n     \"\"\"\n     Implements the CXR-BERT model outlined in the manuscript:\n     Boecking et al. \"Making the Most of Text Semantics to Improve Biomedical Vision-Language Processing\", 2022\n-    https://arxiv.org/abs/2204.09817\n+    https://link.springer.com/chapter/10.1007/978-3-031-20059-5_1\n \n     Extends the HuggingFace BertForMaskedLM model by adding a separate projection head. The projection \"[CLS]\" token is\n     used to align the latent vectors of image and text modalities.\n", "issue": "Links to BioViL paper need to be updated\nWe are still using the arXiv preprint. But this should be solved first:\r\n- https://github.com/microsoft/hi-ml/pull/730#issuecomment-1419298653\n", "before_files": [{"content": "#  ------------------------------------------------------------------------------------------\n#  Copyright (c) Microsoft Corporation. All rights reserved.\n#  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.\n#  ------------------------------------------------------------------------------------------\n\nfrom typing import Any, Optional, Tuple, Union\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nfrom torch import Tensor as T\nfrom transformers import BertForMaskedLM\nfrom transformers.modeling_outputs import ModelOutput\n\nfrom health_multimodal.text.model.configuration_cxrbert import CXRBertConfig\n\nBERTTupleOutput = Tuple[T, T, T, T, T]\n\n\nclass CXRBertOutput(ModelOutput):\n    last_hidden_state: torch.FloatTensor\n    logits: torch.FloatTensor\n    cls_projected_embedding: Optional[torch.FloatTensor] = None\n    hidden_states: Optional[Tuple[torch.FloatTensor]] = None\n    attentions: Optional[Tuple[torch.FloatTensor]] = None\n\n\nclass BertProjectionHead(nn.Module):\n    \"\"\"Projection head to be used with BERT CLS token.\n\n    This is similar to ``BertPredictionHeadTransform`` in HuggingFace.\n\n    :param config: Configuration for BERT.\n    \"\"\"\n\n    def __init__(self, config: CXRBertConfig) -> None:\n        super().__init__()\n        self.dense_to_hidden = nn.Linear(config.hidden_size, config.projection_size)\n        self.transform_act_fn = nn.functional.gelu\n        self.LayerNorm = nn.LayerNorm(config.projection_size, eps=1e-12)\n        self.dense_to_output = nn.Linear(config.projection_size, config.projection_size)\n\n    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:\n        hidden_states = self.dense_to_hidden(hidden_states)\n        hidden_states = self.transform_act_fn(hidden_states)\n        hidden_states = self.LayerNorm(hidden_states)\n        hidden_states = self.dense_to_output(hidden_states)\n\n        return hidden_states\n\n\nclass CXRBertModel(BertForMaskedLM):\n    \"\"\"\n    Implements the CXR-BERT model outlined in the manuscript:\n    Boecking et al. \"Making the Most of Text Semantics to Improve Biomedical Vision-Language Processing\", 2022\n    https://arxiv.org/abs/2204.09817\n\n    Extends the HuggingFace BertForMaskedLM model by adding a separate projection head. The projection \"[CLS]\" token is\n    used to align the latent vectors of image and text modalities.\n    \"\"\"\n\n    config_class = CXRBertConfig  # type: ignore\n\n    def __init__(self, config: CXRBertConfig):\n        super().__init__(config)\n\n        self.cls_projection_head = BertProjectionHead(config)\n        self.init_weights()\n\n    def forward(\n        self,\n        input_ids: torch.Tensor,\n        attention_mask: torch.Tensor,\n        token_type_ids: Optional[torch.Tensor] = None,\n        position_ids: Optional[torch.Tensor] = None,\n        head_mask: Optional[torch.Tensor] = None,\n        inputs_embeds: Optional[torch.Tensor] = None,\n        output_attentions: Optional[bool] = None,\n        output_hidden_states: Optional[bool] = None,\n        output_cls_projected_embedding: Optional[bool] = None,\n        return_dict: Optional[bool] = None,\n        **kwargs: Any\n    ) -> Union[BERTTupleOutput, CXRBertOutput]:\n\n        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n\n        bert_for_masked_lm_output = super().forward(input_ids=input_ids,\n                                                    attention_mask=attention_mask,\n                                                    token_type_ids=token_type_ids,\n                                                    position_ids=position_ids,\n                                                    head_mask=head_mask,\n                                                    inputs_embeds=inputs_embeds,\n                                                    output_attentions=output_attentions,\n                                                    output_hidden_states=True,\n                                                    return_dict=True)\n\n        last_hidden_state = bert_for_masked_lm_output.hidden_states[-1]\n        cls_projected_embedding = self.cls_projection_head(\n            last_hidden_state[:, 0, :]) if output_cls_projected_embedding else None\n\n        if return_dict:\n            return CXRBertOutput(\n                last_hidden_state=last_hidden_state,\n                logits=bert_for_masked_lm_output.logits,\n                cls_projected_embedding=cls_projected_embedding,\n                hidden_states=bert_for_masked_lm_output.hidden_states if output_hidden_states else None,\n                attentions=bert_for_masked_lm_output.attentions,\n            )\n        else:\n            return (\n                last_hidden_state,\n                bert_for_masked_lm_output.logits,\n                cls_projected_embedding,\n                bert_for_masked_lm_output.hidden_states,\n                bert_for_masked_lm_output.attentions,)\n\n    def get_projected_text_embeddings(self,\n                                      input_ids: torch.Tensor,\n                                      attention_mask: torch.Tensor,\n                                      normalize_embeddings: bool = True) -> torch.Tensor:\n        \"\"\"\n        Returns l2-normalised projected cls token embeddings for the given input token ids and attention mask.\n        The joint latent space is trained using a contrastive objective between image and text data modalities.\n\n        :param input_ids: (batch_size, sequence_length)\n        :param attention_mask: (batch_size, sequence_length)\n        :param normalize_embeddings: Whether to l2-normalise the embeddings.\n        :return: (batch_size, projection_size)\n        \"\"\"\n\n        outputs = self.forward(input_ids=input_ids, attention_mask=attention_mask,\n                               output_cls_projected_embedding=True, return_dict=True)\n        assert isinstance(outputs, CXRBertOutput)\n\n        cls_projected_embedding = outputs.cls_projected_embedding\n        assert cls_projected_embedding is not None\n\n        if normalize_embeddings:\n            return F.normalize(cls_projected_embedding, dim=1)\n\n        return cls_projected_embedding\n", "path": "hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py"}], "after_files": [{"content": "#  ------------------------------------------------------------------------------------------\n#  Copyright (c) Microsoft Corporation. All rights reserved.\n#  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.\n#  ------------------------------------------------------------------------------------------\n\nfrom typing import Any, Optional, Tuple, Union\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nfrom torch import Tensor as T\nfrom transformers import BertForMaskedLM\nfrom transformers.modeling_outputs import ModelOutput\n\nfrom health_multimodal.text.model.configuration_cxrbert import CXRBertConfig\n\nBERTTupleOutput = Tuple[T, T, T, T, T]\n\n\nclass CXRBertOutput(ModelOutput):\n    last_hidden_state: torch.FloatTensor\n    logits: torch.FloatTensor\n    cls_projected_embedding: Optional[torch.FloatTensor] = None\n    hidden_states: Optional[Tuple[torch.FloatTensor]] = None\n    attentions: Optional[Tuple[torch.FloatTensor]] = None\n\n\nclass BertProjectionHead(nn.Module):\n    \"\"\"Projection head to be used with BERT CLS token.\n\n    This is similar to ``BertPredictionHeadTransform`` in HuggingFace.\n\n    :param config: Configuration for BERT.\n    \"\"\"\n\n    def __init__(self, config: CXRBertConfig) -> None:\n        super().__init__()\n        self.dense_to_hidden = nn.Linear(config.hidden_size, config.projection_size)\n        self.transform_act_fn = nn.functional.gelu\n        self.LayerNorm = nn.LayerNorm(config.projection_size, eps=1e-12)\n        self.dense_to_output = nn.Linear(config.projection_size, config.projection_size)\n\n    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:\n        hidden_states = self.dense_to_hidden(hidden_states)\n        hidden_states = self.transform_act_fn(hidden_states)\n        hidden_states = self.LayerNorm(hidden_states)\n        hidden_states = self.dense_to_output(hidden_states)\n\n        return hidden_states\n\n\nclass CXRBertModel(BertForMaskedLM):\n    \"\"\"\n    Implements the CXR-BERT model outlined in the manuscript:\n    Boecking et al. \"Making the Most of Text Semantics to Improve Biomedical Vision-Language Processing\", 2022\n    https://link.springer.com/chapter/10.1007/978-3-031-20059-5_1\n\n    Extends the HuggingFace BertForMaskedLM model by adding a separate projection head. The projection \"[CLS]\" token is\n    used to align the latent vectors of image and text modalities.\n    \"\"\"\n\n    config_class = CXRBertConfig  # type: ignore\n\n    def __init__(self, config: CXRBertConfig):\n        super().__init__(config)\n\n        self.cls_projection_head = BertProjectionHead(config)\n        self.init_weights()\n\n    def forward(\n        self,\n        input_ids: torch.Tensor,\n        attention_mask: torch.Tensor,\n        token_type_ids: Optional[torch.Tensor] = None,\n        position_ids: Optional[torch.Tensor] = None,\n        head_mask: Optional[torch.Tensor] = None,\n        inputs_embeds: Optional[torch.Tensor] = None,\n        output_attentions: Optional[bool] = None,\n        output_hidden_states: Optional[bool] = None,\n        output_cls_projected_embedding: Optional[bool] = None,\n        return_dict: Optional[bool] = None,\n        **kwargs: Any\n    ) -> Union[BERTTupleOutput, CXRBertOutput]:\n\n        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n\n        bert_for_masked_lm_output = super().forward(input_ids=input_ids,\n                                                    attention_mask=attention_mask,\n                                                    token_type_ids=token_type_ids,\n                                                    position_ids=position_ids,\n                                                    head_mask=head_mask,\n                                                    inputs_embeds=inputs_embeds,\n                                                    output_attentions=output_attentions,\n                                                    output_hidden_states=True,\n                                                    return_dict=True)\n\n        last_hidden_state = bert_for_masked_lm_output.hidden_states[-1]\n        cls_projected_embedding = self.cls_projection_head(\n            last_hidden_state[:, 0, :]) if output_cls_projected_embedding else None\n\n        if return_dict:\n            return CXRBertOutput(\n                last_hidden_state=last_hidden_state,\n                logits=bert_for_masked_lm_output.logits,\n                cls_projected_embedding=cls_projected_embedding,\n                hidden_states=bert_for_masked_lm_output.hidden_states if output_hidden_states else None,\n                attentions=bert_for_masked_lm_output.attentions,\n            )\n        else:\n            return (\n                last_hidden_state,\n                bert_for_masked_lm_output.logits,\n                cls_projected_embedding,\n                bert_for_masked_lm_output.hidden_states,\n                bert_for_masked_lm_output.attentions,)\n\n    def get_projected_text_embeddings(self,\n                                      input_ids: torch.Tensor,\n                                      attention_mask: torch.Tensor,\n                                      normalize_embeddings: bool = True) -> torch.Tensor:\n        \"\"\"\n        Returns l2-normalised projected cls token embeddings for the given input token ids and attention mask.\n        The joint latent space is trained using a contrastive objective between image and text data modalities.\n\n        :param input_ids: (batch_size, sequence_length)\n        :param attention_mask: (batch_size, sequence_length)\n        :param normalize_embeddings: Whether to l2-normalise the embeddings.\n        :return: (batch_size, projection_size)\n        \"\"\"\n\n        outputs = self.forward(input_ids=input_ids, attention_mask=attention_mask,\n                               output_cls_projected_embedding=True, return_dict=True)\n        assert isinstance(outputs, CXRBertOutput)\n\n        cls_projected_embedding = outputs.cls_projected_embedding\n        assert cls_projected_embedding is not None\n\n        if normalize_embeddings:\n            return F.normalize(cls_projected_embedding, dim=1)\n\n        return cls_projected_embedding\n", "path": "hi-ml-multimodal/src/health_multimodal/text/model/modelling_cxrbert.py"}]}