We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Depth normalization inconsistent for packed vs. nonpacked samples
**Describe the bug**
When the raymarching samples are packed, the depth is calculated according to `sum_i w_i t_i`: https://github.com/nerfstudio-project/nerfstudio/blob/863fc77ab5f247ff3ce3c80f192173063529b036/nerfstudio/model_components/renderers.py#L236

When the raymarching samples are not packed, the depth is calculated with a normalization factor dividing by the total accumulation, `(sum_i w_i t_i) /  (sum_i w_i)`: https://github.com/nerfstudio-project/nerfstudio/blob/863fc77ab5f247ff3ce3c80f192173063529b036/nerfstudio/model_components/renderers.py#L238

**To Reproduce**
N/A

**Expected behavior**
For consistency, the calculation for packed samples should also divide by the total accumulation.

**Screenshots**
N/A

**Additional context**
If this is desired, I can implement the change.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nerfstudio/model_components/renderers.py`
Content:
```
1 # Copyright 2022 The Nerfstudio Team. All rights reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 """
16 Collection of renderers
17 
18 Example:
19 
20 .. code-block:: python
21 
22     field_outputs = field(ray_sampler)
23     weights = ray_sampler.get_weights(field_outputs[FieldHeadNames.DENSITY])
24 
25     rgb_renderer = RGBRenderer()
26     rgb = rgb_renderer(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
27 
28 """
29 import math
30 from typing import Optional, Union
31 
32 import nerfacc
33 import torch
34 from torch import nn
35 from torchtyping import TensorType
36 from typing_extensions import Literal
37 
38 from nerfstudio.cameras.rays import RaySamples
39 from nerfstudio.utils.math import components_from_spherical_harmonics
40 
41 
42 class RGBRenderer(nn.Module):
43     """Standard volumetic rendering.
44 
45     Args:
46         background_color: Background color as RGB. Uses random colors if None.
47     """
48 
49     def __init__(self, background_color: Union[Literal["random", "last_sample"], TensorType[3]] = "random") -> None:
50         super().__init__()
51         self.background_color = background_color
52 
53     @classmethod
54     def combine_rgb(
55         cls,
56         rgb: TensorType["bs":..., "num_samples", 3],
57         weights: TensorType["bs":..., "num_samples", 1],
58         background_color: Union[Literal["random", "last_sample"], TensorType[3]] = "random",
59         ray_indices: Optional[TensorType["num_samples"]] = None,
60         num_rays: Optional[int] = None,
61     ) -> TensorType["bs":..., 3]:
62         """Composite samples along ray and render color image
63 
64         Args:
65             rgb: RGB for each sample
66             weights: Weights for each sample
67             background_color: Background color as RGB.
68             ray_indices: Ray index for each sample, used when samples are packed.
69             num_rays: Number of rays, used when samples are packed.
70 
71         Returns:
72             Outputs rgb values.
73         """
74         if ray_indices is not None and num_rays is not None:
75             # Necessary for packed samples from volumetric ray sampler
76             if background_color == "last_sample":
77                 raise NotImplementedError("Background color 'last_sample' not implemented for packed samples.")
78             comp_rgb = nerfacc.accumulate_along_rays(weights, ray_indices, rgb, num_rays)
79             accumulated_weight = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)
80         else:
81             comp_rgb = torch.sum(weights * rgb, dim=-2)
82             accumulated_weight = torch.sum(weights, dim=-2)
83 
84         if background_color == "last_sample":
85             background_color = rgb[..., -1, :]
86         if background_color == "random":
87             background_color = torch.rand_like(comp_rgb).to(rgb.device)
88 
89         assert isinstance(background_color, torch.Tensor)
90         comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)
91 
92         return comp_rgb
93 
94     def forward(
95         self,
96         rgb: TensorType["bs":..., "num_samples", 3],
97         weights: TensorType["bs":..., "num_samples", 1],
98         ray_indices: Optional[TensorType["num_samples"]] = None,
99         num_rays: Optional[int] = None,
100     ) -> TensorType["bs":..., 3]:
101         """Composite samples along ray and render color image
102 
103         Args:
104             rgb: RGB for each sample
105             weights: Weights for each sample
106             ray_indices: Ray index for each sample, used when samples are packed.
107             num_rays: Number of rays, used when samples are packed.
108 
109         Returns:
110             Outputs of rgb values.
111         """
112 
113         rgb = self.combine_rgb(
114             rgb, weights, background_color=self.background_color, ray_indices=ray_indices, num_rays=num_rays
115         )
116         if not self.training:
117             torch.clamp_(rgb, min=0.0, max=1.0)
118         return rgb
119 
120 
121 class SHRenderer(nn.Module):
122     """Render RGB value from spherical harmonics.
123 
124     Args:
125         background_color: Background color as RGB. Uses random colors if None
126         activation: Output activation.
127     """
128 
129     def __init__(
130         self,
131         background_color: Union[Literal["random", "last_sample"], TensorType[3]] = "random",
132         activation: Optional[nn.Module] = nn.Sigmoid(),
133     ) -> None:
134         super().__init__()
135         self.background_color = background_color
136         self.activation = activation
137 
138     def forward(
139         self,
140         sh: TensorType[..., "num_samples", "coeffs"],
141         directions: TensorType[..., "num_samples", 3],
142         weights: TensorType[..., "num_samples", 1],
143     ) -> TensorType[..., 3]:
144         """Composite samples along ray and render color image
145 
146         Args:
147             sh: Spherical hamonics coefficients for each sample
148             directions: Sample direction
149             weights: Weights for each sample
150 
151         Returns:
152             Outputs of rgb values.
153         """
154 
155         sh = sh.view(*sh.shape[:-1], 3, sh.shape[-1] // 3)
156 
157         levels = int(math.sqrt(sh.shape[-1]))
158         components = components_from_spherical_harmonics(levels=levels, directions=directions)
159 
160         rgb = sh * components[..., None, :]  # [..., num_samples, 3, sh_components]
161         rgb = torch.sum(sh, dim=-1) + 0.5  # [..., num_samples, 3]
162 
163         if self.activation is not None:
164             self.activation(rgb)
165 
166         rgb = RGBRenderer.combine_rgb(rgb, weights, background_color=self.background_color)
167 
168         return rgb
169 
170 
171 class AccumulationRenderer(nn.Module):
172     """Accumulated value along a ray."""
173 
174     @classmethod
175     def forward(
176         cls,
177         weights: TensorType["bs":..., "num_samples", 1],
178         ray_indices: Optional[TensorType["num_samples"]] = None,
179         num_rays: Optional[int] = None,
180     ) -> TensorType["bs":..., 1]:
181         """Composite samples along ray and calculate accumulation.
182 
183         Args:
184             weights: Weights for each sample
185             ray_indices: Ray index for each sample, used when samples are packed.
186             num_rays: Number of rays, used when samples are packed.
187 
188         Returns:
189             Outputs of accumulated values.
190         """
191 
192         if ray_indices is not None and num_rays is not None:
193             # Necessary for packed samples from volumetric ray sampler
194             accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)
195         else:
196             accumulation = torch.sum(weights, dim=-2)
197         return accumulation
198 
199 
200 class DepthRenderer(nn.Module):
201     """Calculate depth along ray.
202 
203     Args:
204         method (str, optional): Depth calculation method.
205     """
206 
207     def __init__(self, method: Literal["expected"] = "expected") -> None:
208         super().__init__()
209         self.method = method
210 
211     def forward(
212         self,
213         weights: TensorType[..., "num_samples", 1],
214         ray_samples: RaySamples,
215         ray_indices: Optional[TensorType["num_samples"]] = None,
216         num_rays: Optional[int] = None,
217     ) -> TensorType[..., 1]:
218         """Composite samples along ray and calculate disparities.
219 
220         Args:
221             weights: Weights for each sample.
222             ray_samples: Set of ray samples.
223             ray_indices: Ray index for each sample, used when samples are packed.
224             num_rays: Number of rays, used when samples are packed.
225 
226         Returns:
227             Outputs of depth values.
228         """
229 
230         if self.method == "expected":
231             eps = 1e-10
232             steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2
233 
234             if ray_indices is not None and num_rays is not None:
235                 # Necessary for packed samples from volumetric ray sampler
236                 depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays)
237             else:
238                 depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps)
239 
240             depth = torch.clip(depth, steps.min(), steps.max())
241 
242             return depth
243 
244         raise NotImplementedError(f"Method {self.method} not implemented")
245 
246 
247 class UncertaintyRenderer(nn.Module):
248     """Calculate uncertainty along the ray."""
249 
250     @classmethod
251     def forward(
252         cls, betas: TensorType["bs":..., "num_samples", 1], weights: TensorType["bs":..., "num_samples", 1]
253     ) -> TensorType["bs":..., 1]:
254         """Calculate uncertainty along the ray.
255 
256         Args:
257             betas: Uncertainty betas for each sample.
258             weights: Weights of each sample.
259 
260         Returns:
261             Rendering of uncertainty.
262         """
263         uncertainty = torch.sum(weights * betas, dim=-2)
264         return uncertainty
265 
266 
267 class SemanticRenderer(nn.Module):
268     """Calculate semantics along the ray."""
269 
270     @classmethod
271     def forward(
272         cls,
273         semantics: TensorType["bs":..., "num_samples", "num_classes"],
274         weights: TensorType["bs":..., "num_samples", 1],
275     ) -> TensorType["bs":..., "num_classes"]:
276         """_summary_"""
277         sem = torch.sum(weights * semantics, dim=-2)
278         return sem
279
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py
--- a/nerfstudio/model_components/renderers.py
+++ b/nerfstudio/model_components/renderers.py
@@ -234,6 +234,8 @@
             if ray_indices is not None and num_rays is not None:
                 # Necessary for packed samples from volumetric ray sampler
                 depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays)
+                accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)
+                depth = depth / (accumulation + eps)
             else:
                 depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps)

{"golden_diff": "diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py\n--- a/nerfstudio/model_components/renderers.py\n+++ b/nerfstudio/model_components/renderers.py\n@@ -234,6 +234,8 @@\n             if ray_indices is not None and num_rays is not None:\n                 # Necessary for packed samples from volumetric ray sampler\n                 depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays)\n+                accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)\n+                depth = depth / (accumulation + eps)\n             else:\n                 depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps)\n", "issue": "Depth normalization inconsistent for packed vs. nonpacked samples\n**Describe the bug**\r\nWhen the raymarching samples are packed, the depth is calculated according to `sum_i w_i t_i`: https://github.com/nerfstudio-project/nerfstudio/blob/863fc77ab5f247ff3ce3c80f192173063529b036/nerfstudio/model_components/renderers.py#L236\r\n\r\nWhen the raymarching samples are not packed, the depth is calculated with a normalization factor dividing by the total accumulation, `(sum_i w_i t_i) /  (sum_i w_i)`: https://github.com/nerfstudio-project/nerfstudio/blob/863fc77ab5f247ff3ce3c80f192173063529b036/nerfstudio/model_components/renderers.py#L238\r\n\r\n**To Reproduce**\r\nN/A\r\n\r\n**Expected behavior**\r\nFor consistency, the calculation for packed samples should also divide by the total accumulation.\r\n\r\n**Screenshots**\r\nN/A\r\n\r\n**Additional context**\r\nIf this is desired, I can implement the change.\r\n\n", "before_files": [{"content": "# Copyright 2022 The Nerfstudio Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nCollection of renderers\n\nExample:\n\n.. code-block:: python\n\n    field_outputs = field(ray_sampler)\n    weights = ray_sampler.get_weights(field_outputs[FieldHeadNames.DENSITY])\n\n    rgb_renderer = RGBRenderer()\n    rgb = rgb_renderer(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)\n\n\"\"\"\nimport math\nfrom typing import Optional, Union\n\nimport nerfacc\nimport torch\nfrom torch import nn\nfrom torchtyping import TensorType\nfrom typing_extensions import Literal\n\nfrom nerfstudio.cameras.rays import RaySamples\nfrom nerfstudio.utils.math import components_from_spherical_harmonics\n\n\nclass RGBRenderer(nn.Module):\n    \"\"\"Standard volumetic rendering.\n\n    Args:\n        background_color: Background color as RGB. Uses random colors if None.\n    \"\"\"\n\n    def __init__(self, background_color: Union[Literal[\"random\", \"last_sample\"], TensorType[3]] = \"random\") -> None:\n        super().__init__()\n        self.background_color = background_color\n\n    @classmethod\n    def combine_rgb(\n        cls,\n        rgb: TensorType[\"bs\":..., \"num_samples\", 3],\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n        background_color: Union[Literal[\"random\", \"last_sample\"], TensorType[3]] = \"random\",\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[\"bs\":..., 3]:\n        \"\"\"Composite samples along ray and render color image\n\n        Args:\n            rgb: RGB for each sample\n            weights: Weights for each sample\n            background_color: Background color as RGB.\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs rgb values.\n        \"\"\"\n        if ray_indices is not None and num_rays is not None:\n            # Necessary for packed samples from volumetric ray sampler\n            if background_color == \"last_sample\":\n                raise NotImplementedError(\"Background color 'last_sample' not implemented for packed samples.\")\n            comp_rgb = nerfacc.accumulate_along_rays(weights, ray_indices, rgb, num_rays)\n            accumulated_weight = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)\n        else:\n            comp_rgb = torch.sum(weights * rgb, dim=-2)\n            accumulated_weight = torch.sum(weights, dim=-2)\n\n        if background_color == \"last_sample\":\n            background_color = rgb[..., -1, :]\n        if background_color == \"random\":\n            background_color = torch.rand_like(comp_rgb).to(rgb.device)\n\n        assert isinstance(background_color, torch.Tensor)\n        comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)\n\n        return comp_rgb\n\n    def forward(\n        self,\n        rgb: TensorType[\"bs\":..., \"num_samples\", 3],\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[\"bs\":..., 3]:\n        \"\"\"Composite samples along ray and render color image\n\n        Args:\n            rgb: RGB for each sample\n            weights: Weights for each sample\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs of rgb values.\n        \"\"\"\n\n        rgb = self.combine_rgb(\n            rgb, weights, background_color=self.background_color, ray_indices=ray_indices, num_rays=num_rays\n        )\n        if not self.training:\n            torch.clamp_(rgb, min=0.0, max=1.0)\n        return rgb\n\n\nclass SHRenderer(nn.Module):\n    \"\"\"Render RGB value from spherical harmonics.\n\n    Args:\n        background_color: Background color as RGB. Uses random colors if None\n        activation: Output activation.\n    \"\"\"\n\n    def __init__(\n        self,\n        background_color: Union[Literal[\"random\", \"last_sample\"], TensorType[3]] = \"random\",\n        activation: Optional[nn.Module] = nn.Sigmoid(),\n    ) -> None:\n        super().__init__()\n        self.background_color = background_color\n        self.activation = activation\n\n    def forward(\n        self,\n        sh: TensorType[..., \"num_samples\", \"coeffs\"],\n        directions: TensorType[..., \"num_samples\", 3],\n        weights: TensorType[..., \"num_samples\", 1],\n    ) -> TensorType[..., 3]:\n        \"\"\"Composite samples along ray and render color image\n\n        Args:\n            sh: Spherical hamonics coefficients for each sample\n            directions: Sample direction\n            weights: Weights for each sample\n\n        Returns:\n            Outputs of rgb values.\n        \"\"\"\n\n        sh = sh.view(*sh.shape[:-1], 3, sh.shape[-1] // 3)\n\n        levels = int(math.sqrt(sh.shape[-1]))\n        components = components_from_spherical_harmonics(levels=levels, directions=directions)\n\n        rgb = sh * components[..., None, :]  # [..., num_samples, 3, sh_components]\n        rgb = torch.sum(sh, dim=-1) + 0.5  # [..., num_samples, 3]\n\n        if self.activation is not None:\n            self.activation(rgb)\n\n        rgb = RGBRenderer.combine_rgb(rgb, weights, background_color=self.background_color)\n\n        return rgb\n\n\nclass AccumulationRenderer(nn.Module):\n    \"\"\"Accumulated value along a ray.\"\"\"\n\n    @classmethod\n    def forward(\n        cls,\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[\"bs\":..., 1]:\n        \"\"\"Composite samples along ray and calculate accumulation.\n\n        Args:\n            weights: Weights for each sample\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs of accumulated values.\n        \"\"\"\n\n        if ray_indices is not None and num_rays is not None:\n            # Necessary for packed samples from volumetric ray sampler\n            accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)\n        else:\n            accumulation = torch.sum(weights, dim=-2)\n        return accumulation\n\n\nclass DepthRenderer(nn.Module):\n    \"\"\"Calculate depth along ray.\n\n    Args:\n        method (str, optional): Depth calculation method.\n    \"\"\"\n\n    def __init__(self, method: Literal[\"expected\"] = \"expected\") -> None:\n        super().__init__()\n        self.method = method\n\n    def forward(\n        self,\n        weights: TensorType[..., \"num_samples\", 1],\n        ray_samples: RaySamples,\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[..., 1]:\n        \"\"\"Composite samples along ray and calculate disparities.\n\n        Args:\n            weights: Weights for each sample.\n            ray_samples: Set of ray samples.\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs of depth values.\n        \"\"\"\n\n        if self.method == \"expected\":\n            eps = 1e-10\n            steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2\n\n            if ray_indices is not None and num_rays is not None:\n                # Necessary for packed samples from volumetric ray sampler\n                depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays)\n            else:\n                depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps)\n\n            depth = torch.clip(depth, steps.min(), steps.max())\n\n            return depth\n\n        raise NotImplementedError(f\"Method {self.method} not implemented\")\n\n\nclass UncertaintyRenderer(nn.Module):\n    \"\"\"Calculate uncertainty along the ray.\"\"\"\n\n    @classmethod\n    def forward(\n        cls, betas: TensorType[\"bs\":..., \"num_samples\", 1], weights: TensorType[\"bs\":..., \"num_samples\", 1]\n    ) -> TensorType[\"bs\":..., 1]:\n        \"\"\"Calculate uncertainty along the ray.\n\n        Args:\n            betas: Uncertainty betas for each sample.\n            weights: Weights of each sample.\n\n        Returns:\n            Rendering of uncertainty.\n        \"\"\"\n        uncertainty = torch.sum(weights * betas, dim=-2)\n        return uncertainty\n\n\nclass SemanticRenderer(nn.Module):\n    \"\"\"Calculate semantics along the ray.\"\"\"\n\n    @classmethod\n    def forward(\n        cls,\n        semantics: TensorType[\"bs\":..., \"num_samples\", \"num_classes\"],\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n    ) -> TensorType[\"bs\":..., \"num_classes\"]:\n        \"\"\"_summary_\"\"\"\n        sem = torch.sum(weights * semantics, dim=-2)\n        return sem\n", "path": "nerfstudio/model_components/renderers.py"}], "after_files": [{"content": "# Copyright 2022 The Nerfstudio Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nCollection of renderers\n\nExample:\n\n.. code-block:: python\n\n    field_outputs = field(ray_sampler)\n    weights = ray_sampler.get_weights(field_outputs[FieldHeadNames.DENSITY])\n\n    rgb_renderer = RGBRenderer()\n    rgb = rgb_renderer(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)\n\n\"\"\"\nimport math\nfrom typing import Optional, Union\n\nimport nerfacc\nimport torch\nfrom torch import nn\nfrom torchtyping import TensorType\nfrom typing_extensions import Literal\n\nfrom nerfstudio.cameras.rays import RaySamples\nfrom nerfstudio.utils.math import components_from_spherical_harmonics\n\n\nclass RGBRenderer(nn.Module):\n    \"\"\"Standard volumetic rendering.\n\n    Args:\n        background_color: Background color as RGB. Uses random colors if None.\n    \"\"\"\n\n    def __init__(self, background_color: Union[Literal[\"random\", \"last_sample\"], TensorType[3]] = \"random\") -> None:\n        super().__init__()\n        self.background_color = background_color\n\n    @classmethod\n    def combine_rgb(\n        cls,\n        rgb: TensorType[\"bs\":..., \"num_samples\", 3],\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n        background_color: Union[Literal[\"random\", \"last_sample\"], TensorType[3]] = \"random\",\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[\"bs\":..., 3]:\n        \"\"\"Composite samples along ray and render color image\n\n        Args:\n            rgb: RGB for each sample\n            weights: Weights for each sample\n            background_color: Background color as RGB.\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs rgb values.\n        \"\"\"\n        if ray_indices is not None and num_rays is not None:\n            # Necessary for packed samples from volumetric ray sampler\n            if background_color == \"last_sample\":\n                raise NotImplementedError(\"Background color 'last_sample' not implemented for packed samples.\")\n            comp_rgb = nerfacc.accumulate_along_rays(weights, ray_indices, rgb, num_rays)\n            accumulated_weight = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)\n        else:\n            comp_rgb = torch.sum(weights * rgb, dim=-2)\n            accumulated_weight = torch.sum(weights, dim=-2)\n\n        if background_color == \"last_sample\":\n            background_color = rgb[..., -1, :]\n        if background_color == \"random\":\n            background_color = torch.rand_like(comp_rgb).to(rgb.device)\n\n        assert isinstance(background_color, torch.Tensor)\n        comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)\n\n        return comp_rgb\n\n    def forward(\n        self,\n        rgb: TensorType[\"bs\":..., \"num_samples\", 3],\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[\"bs\":..., 3]:\n        \"\"\"Composite samples along ray and render color image\n\n        Args:\n            rgb: RGB for each sample\n            weights: Weights for each sample\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs of rgb values.\n        \"\"\"\n\n        rgb = self.combine_rgb(\n            rgb, weights, background_color=self.background_color, ray_indices=ray_indices, num_rays=num_rays\n        )\n        if not self.training:\n            torch.clamp_(rgb, min=0.0, max=1.0)\n        return rgb\n\n\nclass SHRenderer(nn.Module):\n    \"\"\"Render RGB value from spherical harmonics.\n\n    Args:\n        background_color: Background color as RGB. Uses random colors if None\n        activation: Output activation.\n    \"\"\"\n\n    def __init__(\n        self,\n        background_color: Union[Literal[\"random\", \"last_sample\"], TensorType[3]] = \"random\",\n        activation: Optional[nn.Module] = nn.Sigmoid(),\n    ) -> None:\n        super().__init__()\n        self.background_color = background_color\n        self.activation = activation\n\n    def forward(\n        self,\n        sh: TensorType[..., \"num_samples\", \"coeffs\"],\n        directions: TensorType[..., \"num_samples\", 3],\n        weights: TensorType[..., \"num_samples\", 1],\n    ) -> TensorType[..., 3]:\n        \"\"\"Composite samples along ray and render color image\n\n        Args:\n            sh: Spherical hamonics coefficients for each sample\n            directions: Sample direction\n            weights: Weights for each sample\n\n        Returns:\n            Outputs of rgb values.\n        \"\"\"\n\n        sh = sh.view(*sh.shape[:-1], 3, sh.shape[-1] // 3)\n\n        levels = int(math.sqrt(sh.shape[-1]))\n        components = components_from_spherical_harmonics(levels=levels, directions=directions)\n\n        rgb = sh * components[..., None, :]  # [..., num_samples, 3, sh_components]\n        rgb = torch.sum(sh, dim=-1) + 0.5  # [..., num_samples, 3]\n\n        if self.activation is not None:\n            self.activation(rgb)\n\n        rgb = RGBRenderer.combine_rgb(rgb, weights, background_color=self.background_color)\n\n        return rgb\n\n\nclass AccumulationRenderer(nn.Module):\n    \"\"\"Accumulated value along a ray.\"\"\"\n\n    @classmethod\n    def forward(\n        cls,\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[\"bs\":..., 1]:\n        \"\"\"Composite samples along ray and calculate accumulation.\n\n        Args:\n            weights: Weights for each sample\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs of accumulated values.\n        \"\"\"\n\n        if ray_indices is not None and num_rays is not None:\n            # Necessary for packed samples from volumetric ray sampler\n            accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)\n        else:\n            accumulation = torch.sum(weights, dim=-2)\n        return accumulation\n\n\nclass DepthRenderer(nn.Module):\n    \"\"\"Calculate depth along ray.\n\n    Args:\n        method (str, optional): Depth calculation method.\n    \"\"\"\n\n    def __init__(self, method: Literal[\"expected\"] = \"expected\") -> None:\n        super().__init__()\n        self.method = method\n\n    def forward(\n        self,\n        weights: TensorType[..., \"num_samples\", 1],\n        ray_samples: RaySamples,\n        ray_indices: Optional[TensorType[\"num_samples\"]] = None,\n        num_rays: Optional[int] = None,\n    ) -> TensorType[..., 1]:\n        \"\"\"Composite samples along ray and calculate disparities.\n\n        Args:\n            weights: Weights for each sample.\n            ray_samples: Set of ray samples.\n            ray_indices: Ray index for each sample, used when samples are packed.\n            num_rays: Number of rays, used when samples are packed.\n\n        Returns:\n            Outputs of depth values.\n        \"\"\"\n\n        if self.method == \"expected\":\n            eps = 1e-10\n            steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2\n\n            if ray_indices is not None and num_rays is not None:\n                # Necessary for packed samples from volumetric ray sampler\n                depth = nerfacc.accumulate_along_rays(weights, ray_indices, steps, num_rays)\n                accumulation = nerfacc.accumulate_along_rays(weights, ray_indices, None, num_rays)\n                depth = depth / (accumulation + eps)\n            else:\n                depth = torch.sum(weights * steps, dim=-2) / (torch.sum(weights, -2) + eps)\n\n            depth = torch.clip(depth, steps.min(), steps.max())\n\n            return depth\n\n        raise NotImplementedError(f\"Method {self.method} not implemented\")\n\n\nclass UncertaintyRenderer(nn.Module):\n    \"\"\"Calculate uncertainty along the ray.\"\"\"\n\n    @classmethod\n    def forward(\n        cls, betas: TensorType[\"bs\":..., \"num_samples\", 1], weights: TensorType[\"bs\":..., \"num_samples\", 1]\n    ) -> TensorType[\"bs\":..., 1]:\n        \"\"\"Calculate uncertainty along the ray.\n\n        Args:\n            betas: Uncertainty betas for each sample.\n            weights: Weights of each sample.\n\n        Returns:\n            Rendering of uncertainty.\n        \"\"\"\n        uncertainty = torch.sum(weights * betas, dim=-2)\n        return uncertainty\n\n\nclass SemanticRenderer(nn.Module):\n    \"\"\"Calculate semantics along the ray.\"\"\"\n\n    @classmethod\n    def forward(\n        cls,\n        semantics: TensorType[\"bs\":..., \"num_samples\", \"num_classes\"],\n        weights: TensorType[\"bs\":..., \"num_samples\", 1],\n    ) -> TensorType[\"bs\":..., \"num_classes\"]:\n        \"\"\"_summary_\"\"\"\n        sem = torch.sum(weights * semantics, dim=-2)\n        return sem\n", "path": "nerfstudio/model_components/renderers.py"}]}