We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Unexpected weight initialization
Thanks for your error report and we appreciate it a lot.

**Checklist**

1. I have searched related issues but cannot get the expected help.
2. I have read the [FAQ documentation](https://mmengine.readthedocs.io/en/latest/faq.html) but cannot get the expected help.
3. The bug has not been fixed in the latest version.

**Describe the bug**

When using `Pretrained` init_cfg to init a model, the outside init_cfg will **disable all submodules' init_weights**.

If the outside model's pretrained weight does not strictly match, e.g. different num_classes, the mismatched submodule's weight will initialize **by pytorch's default init weight logic**, instead of the submodule's init_cfg. 

This lead's to some unacceptable results, e.g. a huge classification loss due to the failed initialization of the classification head when finetuning a model.


**Reproduction**

In mmdet, finetuning rtmdet_l_8xb32-300e_coco

```python
_base_ = './rtmdet_l_8xb32-300e_coco.py'

checkpoint = 'work_dir/rtmdet/rtmdet_l_8xb32-300e_coco/epoch_300.pth'
model = dict(bbox_head=dict(num_classes=1), init_cfg=dict(type='Pretrained', checkpoint=checkpoint))
```



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mmengine/model/base_module.py`
Content:
```
1 # Copyright (c) OpenMMLab. All rights reserved.
2 import copy
3 import logging
4 import warnings
5 from abc import ABCMeta
6 from collections import defaultdict
7 from logging import FileHandler
8 from typing import Iterable, Optional
9 
10 import torch.nn as nn
11 
12 from mmengine.dist import master_only
13 from mmengine.logging import MMLogger, print_log
14 from .weight_init import initialize, update_init_info
15 
16 
17 class BaseModule(nn.Module, metaclass=ABCMeta):
18     """Base module for all modules in openmmlab. ``BaseModule`` is a wrapper of
19     ``torch.nn.Module`` with additional functionality of parameter
20     initialization. Compared with ``torch.nn.Module``, ``BaseModule`` mainly
21     adds three attributes.
22 
23     - ``init_cfg``: the config to control the initialization.
24     - ``init_weights``: The function of parameter initialization and recording
25       initialization information.
26     - ``_params_init_info``: Used to track the parameter initialization
27       information. This attribute only exists during executing the
28       ``init_weights``.
29     Args:
30         init_cfg (dict, optional): Initialization config dict.
31     """
32 
33     def __init__(self, init_cfg=None):
34         """Initialize BaseModule, inherited from `torch.nn.Module`"""
35 
36         # NOTE init_cfg can be defined in different levels, but init_cfg
37         # in low levels has a higher priority.
38 
39         super().__init__()
40         # define default value of init_cfg instead of hard code
41         # in init_weights() function
42         self._is_init = False
43 
44         self.init_cfg = copy.deepcopy(init_cfg)
45 
46         # Backward compatibility in derived classes
47         # if pretrained is not None:
48         #     warnings.warn('DeprecationWarning: pretrained is a deprecated \
49         #         key, please consider using init_cfg')
50         #     self.init_cfg = dict(type='Pretrained', checkpoint=pretrained)
51 
52     @property
53     def is_init(self):
54         return self._is_init
55 
56     def init_weights(self):
57         """Initialize the weights."""
58 
59         is_top_level_module = False
60         # check if it is top-level module
61         if not hasattr(self, '_params_init_info'):
62             # The `_params_init_info` is used to record the initialization
63             # information of the parameters
64             # the key should be the obj:`nn.Parameter` of model and the value
65             # should be a dict containing
66             # - init_info (str): The string that describes the initialization.
67             # - tmp_mean_value (FloatTensor): The mean of the parameter,
68             #       which indicates whether the parameter has been modified.
69             # this attribute would be deleted after all parameters
70             # is initialized.
71             self._params_init_info = defaultdict(dict)
72             is_top_level_module = True
73 
74             # Initialize the `_params_init_info`,
75             # When detecting the `tmp_mean_value` of
76             # the corresponding parameter is changed, update related
77             # initialization information
78             for name, param in self.named_parameters():
79                 self._params_init_info[param][
80                     'init_info'] = f'The value is the same before and ' \
81                                    f'after calling `init_weights` ' \
82                                    f'of {self.__class__.__name__} '
83                 self._params_init_info[param][
84                     'tmp_mean_value'] = param.data.mean().cpu()
85 
86             # pass `params_init_info` to all submodules
87             # All submodules share the same `params_init_info`,
88             # so it will be updated when parameters are
89             # modified at any level of the model.
90             for sub_module in self.modules():
91                 sub_module._params_init_info = self._params_init_info
92 
93         logger = MMLogger.get_current_instance()
94         logger_name = logger.instance_name
95 
96         module_name = self.__class__.__name__
97         if not self._is_init:
98             if self.init_cfg:
99                 print_log(
100                     f'initialize {module_name} with init_cfg {self.init_cfg}',
101                     logger=logger_name,
102                     level=logging.DEBUG)
103                 initialize(self, self.init_cfg)
104                 if isinstance(self.init_cfg, dict):
105                     # prevent the parameters of
106                     # the pre-trained model
107                     # from being overwritten by
108                     # the `init_weights`
109                     if self.init_cfg['type'] == 'Pretrained':
110                         return
111 
112             for m in self.children():
113                 if hasattr(m, 'init_weights'):
114                     m.init_weights()
115                     # users may overload the `init_weights`
116                     update_init_info(
117                         m,
118                         init_info=f'Initialized by '
119                         f'user-defined `init_weights`'
120                         f' in {m.__class__.__name__} ')
121 
122             self._is_init = True
123         else:
124             warnings.warn(f'init_weights of {self.__class__.__name__} has '
125                           f'been called more than once.')
126 
127         if is_top_level_module:
128             # self._dump_init_info(logger_name)
129             self._dump_init_info()
130 
131             for sub_module in self.modules():
132                 del sub_module._params_init_info
133 
134     @master_only
135     def _dump_init_info(self):
136         """Dump the initialization information to a file named
137         `initialization.log.json` in workdir.
138 
139         Args:
140             logger_name (str): The name of logger.
141         """
142 
143         logger = MMLogger.get_current_instance()
144         logger_name = logger.instance_name
145         with_file_handler = False
146         # dump the information to the logger file if there is a `FileHandler`
147         for handler in logger.handlers:
148             if isinstance(handler, FileHandler):
149                 handler.stream.write(
150                     'Name of parameter - Initialization information\n')
151                 for name, param in self.named_parameters():
152                     handler.stream.write(
153                         f'\n{name} - {param.shape}: '
154                         f"\n{self._params_init_info[param]['init_info']} \n")
155                 handler.stream.flush()
156                 with_file_handler = True
157         if not with_file_handler:
158             for name, param in self.named_parameters():
159                 print_log(
160                     f'\n{name} - {param.shape}: '
161                     f"\n{self._params_init_info[param]['init_info']} \n ",
162                     logger=logger_name)
163 
164     def __repr__(self):
165         s = super().__repr__()
166         if self.init_cfg:
167             s += f'\ninit_cfg={self.init_cfg}'
168         return s
169 
170 
171 class Sequential(BaseModule, nn.Sequential):
172     """Sequential module in openmmlab.
173 
174     Ensures that all modules in ``Sequential`` have a different initialization
175     strategy than the outer model
176 
177     Args:
178         init_cfg (dict, optional): Initialization config dict.
179     """
180 
181     def __init__(self, *args, init_cfg: Optional[dict] = None):
182         BaseModule.__init__(self, init_cfg)
183         nn.Sequential.__init__(self, *args)
184 
185 
186 class ModuleList(BaseModule, nn.ModuleList):
187     """ModuleList in openmmlab.
188 
189     Ensures that all modules in ``ModuleList`` have a different initialization
190     strategy than the outer model
191 
192     Args:
193         modules (iterable, optional): An iterable of modules to add.
194         init_cfg (dict, optional): Initialization config dict.
195     """
196 
197     def __init__(self,
198                  modules: Optional[Iterable] = None,
199                  init_cfg: Optional[dict] = None):
200         BaseModule.__init__(self, init_cfg)
201         nn.ModuleList.__init__(self, modules)
202 
203 
204 class ModuleDict(BaseModule, nn.ModuleDict):
205     """ModuleDict in openmmlab.
206 
207     Ensures that all modules in ``ModuleDict`` have a different initialization
208     strategy than the outer model
209 
210     Args:
211         modules (dict, optional): A mapping (dictionary) of (string: module)
212             or an iterable of key-value pairs of type (string, module).
213         init_cfg (dict, optional): Initialization config dict.
214     """
215 
216     def __init__(self,
217                  modules: Optional[dict] = None,
218                  init_cfg: Optional[dict] = None):
219         BaseModule.__init__(self, init_cfg)
220         nn.ModuleDict.__init__(self, modules)
221
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mmengine/model/base_module.py b/mmengine/model/base_module.py
--- a/mmengine/model/base_module.py
+++ b/mmengine/model/base_module.py
@@ -5,7 +5,7 @@
 from abc import ABCMeta
 from collections import defaultdict
 from logging import FileHandler
-from typing import Iterable, Optional
+from typing import Iterable, List, Optional, Union
 
 import torch.nn as nn
 
@@ -26,11 +26,17 @@
     - ``_params_init_info``: Used to track the parameter initialization
       information. This attribute only exists during executing the
       ``init_weights``.
+
+    Note:
+        :obj:`PretrainedInit` has a higher priority than any other
+        initializer. The loaded pretrained weights will overwrite
+        the previous initialized weights.
+
     Args:
-        init_cfg (dict, optional): Initialization config dict.
+        init_cfg (dict or List[dict], optional): Initialization config dict.
     """
 
-    def __init__(self, init_cfg=None):
+    def __init__(self, init_cfg: Union[dict, List[dict], None] = None):
         """Initialize BaseModule, inherited from `torch.nn.Module`"""
 
         # NOTE init_cfg can be defined in different levels, but init_cfg
@@ -100,14 +106,25 @@
                     f'initialize {module_name} with init_cfg {self.init_cfg}',
                     logger=logger_name,
                     level=logging.DEBUG)
-                initialize(self, self.init_cfg)
+
+                init_cfgs = self.init_cfg
                 if isinstance(self.init_cfg, dict):
-                    # prevent the parameters of
-                    # the pre-trained model
-                    # from being overwritten by
-                    # the `init_weights`
-                    if self.init_cfg['type'] == 'Pretrained':
-                        return
+                    init_cfgs = [self.init_cfg]
+
+                # PretrainedInit has higher priority than any other init_cfg.
+                # Therefore we initialize `pretrained_cfg` last to overwrite
+                # the previous initialized weights.
+                # See details in https://github.com/open-mmlab/mmengine/issues/691 # noqa E501
+                other_cfgs = []
+                pretrained_cfg = []
+                for init_cfg in init_cfgs:
+                    assert isinstance(init_cfg, dict)
+                    if init_cfg['type'] == 'Pretrained':
+                        pretrained_cfg.append(init_cfg)
+                    else:
+                        other_cfgs.append(init_cfg)
+
+                initialize(self, other_cfgs)
 
             for m in self.children():
                 if hasattr(m, 'init_weights'):
@@ -118,7 +135,8 @@
                         init_info=f'Initialized by '
                         f'user-defined `init_weights`'
                         f' in {m.__class__.__name__} ')
-
+            if self.init_cfg and pretrained_cfg:
+                initialize(self, pretrained_cfg)
             self._is_init = True
         else:
             warnings.warn(f'init_weights of {self.__class__.__name__} has '

{"golden_diff": "diff --git a/mmengine/model/base_module.py b/mmengine/model/base_module.py\n--- a/mmengine/model/base_module.py\n+++ b/mmengine/model/base_module.py\n@@ -5,7 +5,7 @@\n from abc import ABCMeta\n from collections import defaultdict\n from logging import FileHandler\n-from typing import Iterable, Optional\n+from typing import Iterable, List, Optional, Union\n \n import torch.nn as nn\n \n@@ -26,11 +26,17 @@\n     - ``_params_init_info``: Used to track the parameter initialization\n       information. This attribute only exists during executing the\n       ``init_weights``.\n+\n+    Note:\n+        :obj:`PretrainedInit` has a higher priority than any other\n+        initializer. The loaded pretrained weights will overwrite\n+        the previous initialized weights.\n+\n     Args:\n-        init_cfg (dict, optional): Initialization config dict.\n+        init_cfg (dict or List[dict], optional): Initialization config dict.\n     \"\"\"\n \n-    def __init__(self, init_cfg=None):\n+    def __init__(self, init_cfg: Union[dict, List[dict], None] = None):\n         \"\"\"Initialize BaseModule, inherited from `torch.nn.Module`\"\"\"\n \n         # NOTE init_cfg can be defined in different levels, but init_cfg\n@@ -100,14 +106,25 @@\n                     f'initialize {module_name} with init_cfg {self.init_cfg}',\n                     logger=logger_name,\n                     level=logging.DEBUG)\n-                initialize(self, self.init_cfg)\n+\n+                init_cfgs = self.init_cfg\n                 if isinstance(self.init_cfg, dict):\n-                    # prevent the parameters of\n-                    # the pre-trained model\n-                    # from being overwritten by\n-                    # the `init_weights`\n-                    if self.init_cfg['type'] == 'Pretrained':\n-                        return\n+                    init_cfgs = [self.init_cfg]\n+\n+                # PretrainedInit has higher priority than any other init_cfg.\n+                # Therefore we initialize `pretrained_cfg` last to overwrite\n+                # the previous initialized weights.\n+                # See details in https://github.com/open-mmlab/mmengine/issues/691 # noqa E501\n+                other_cfgs = []\n+                pretrained_cfg = []\n+                for init_cfg in init_cfgs:\n+                    assert isinstance(init_cfg, dict)\n+                    if init_cfg['type'] == 'Pretrained':\n+                        pretrained_cfg.append(init_cfg)\n+                    else:\n+                        other_cfgs.append(init_cfg)\n+\n+                initialize(self, other_cfgs)\n \n             for m in self.children():\n                 if hasattr(m, 'init_weights'):\n@@ -118,7 +135,8 @@\n                         init_info=f'Initialized by '\n                         f'user-defined `init_weights`'\n                         f' in {m.__class__.__name__} ')\n-\n+            if self.init_cfg and pretrained_cfg:\n+                initialize(self, pretrained_cfg)\n             self._is_init = True\n         else:\n             warnings.warn(f'init_weights of {self.__class__.__name__} has '\n", "issue": "Unexpected weight initialization\nThanks for your error report and we appreciate it a lot.\r\n\r\n**Checklist**\r\n\r\n1. I have searched related issues but cannot get the expected help.\r\n2. I have read the [FAQ documentation](https://mmengine.readthedocs.io/en/latest/faq.html) but cannot get the expected help.\r\n3. The bug has not been fixed in the latest version.\r\n\r\n**Describe the bug**\r\n\r\nWhen using `Pretrained` init_cfg to init a model, the outside init_cfg will **disable all submodules' init_weights**.\r\n\r\nIf the outside model's pretrained weight does not strictly match, e.g. different num_classes, the mismatched submodule's weight will initialize **by pytorch's default init weight logic**, instead of the submodule's init_cfg. \r\n\r\nThis lead's to some unacceptable results, e.g. a huge classification loss due to the failed initialization of the classification head when finetuning a model.\r\n\r\n\r\n**Reproduction**\r\n\r\nIn mmdet, finetuning rtmdet_l_8xb32-300e_coco\r\n\r\n```python\r\n_base_ = './rtmdet_l_8xb32-300e_coco.py'\r\n\r\ncheckpoint = 'work_dir/rtmdet/rtmdet_l_8xb32-300e_coco/epoch_300.pth'\r\nmodel = dict(bbox_head=dict(num_classes=1), init_cfg=dict(type='Pretrained', checkpoint=checkpoint))\r\n```\r\n\r\n\n", "before_files": [{"content": "# Copyright (c) OpenMMLab. All rights reserved.\nimport copy\nimport logging\nimport warnings\nfrom abc import ABCMeta\nfrom collections import defaultdict\nfrom logging import FileHandler\nfrom typing import Iterable, Optional\n\nimport torch.nn as nn\n\nfrom mmengine.dist import master_only\nfrom mmengine.logging import MMLogger, print_log\nfrom .weight_init import initialize, update_init_info\n\n\nclass BaseModule(nn.Module, metaclass=ABCMeta):\n    \"\"\"Base module for all modules in openmmlab. ``BaseModule`` is a wrapper of\n    ``torch.nn.Module`` with additional functionality of parameter\n    initialization. Compared with ``torch.nn.Module``, ``BaseModule`` mainly\n    adds three attributes.\n\n    - ``init_cfg``: the config to control the initialization.\n    - ``init_weights``: The function of parameter initialization and recording\n      initialization information.\n    - ``_params_init_info``: Used to track the parameter initialization\n      information. This attribute only exists during executing the\n      ``init_weights``.\n    Args:\n        init_cfg (dict, optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self, init_cfg=None):\n        \"\"\"Initialize BaseModule, inherited from `torch.nn.Module`\"\"\"\n\n        # NOTE init_cfg can be defined in different levels, but init_cfg\n        # in low levels has a higher priority.\n\n        super().__init__()\n        # define default value of init_cfg instead of hard code\n        # in init_weights() function\n        self._is_init = False\n\n        self.init_cfg = copy.deepcopy(init_cfg)\n\n        # Backward compatibility in derived classes\n        # if pretrained is not None:\n        #     warnings.warn('DeprecationWarning: pretrained is a deprecated \\\n        #         key, please consider using init_cfg')\n        #     self.init_cfg = dict(type='Pretrained', checkpoint=pretrained)\n\n    @property\n    def is_init(self):\n        return self._is_init\n\n    def init_weights(self):\n        \"\"\"Initialize the weights.\"\"\"\n\n        is_top_level_module = False\n        # check if it is top-level module\n        if not hasattr(self, '_params_init_info'):\n            # The `_params_init_info` is used to record the initialization\n            # information of the parameters\n            # the key should be the obj:`nn.Parameter` of model and the value\n            # should be a dict containing\n            # - init_info (str): The string that describes the initialization.\n            # - tmp_mean_value (FloatTensor): The mean of the parameter,\n            #       which indicates whether the parameter has been modified.\n            # this attribute would be deleted after all parameters\n            # is initialized.\n            self._params_init_info = defaultdict(dict)\n            is_top_level_module = True\n\n            # Initialize the `_params_init_info`,\n            # When detecting the `tmp_mean_value` of\n            # the corresponding parameter is changed, update related\n            # initialization information\n            for name, param in self.named_parameters():\n                self._params_init_info[param][\n                    'init_info'] = f'The value is the same before and ' \\\n                                   f'after calling `init_weights` ' \\\n                                   f'of {self.__class__.__name__} '\n                self._params_init_info[param][\n                    'tmp_mean_value'] = param.data.mean().cpu()\n\n            # pass `params_init_info` to all submodules\n            # All submodules share the same `params_init_info`,\n            # so it will be updated when parameters are\n            # modified at any level of the model.\n            for sub_module in self.modules():\n                sub_module._params_init_info = self._params_init_info\n\n        logger = MMLogger.get_current_instance()\n        logger_name = logger.instance_name\n\n        module_name = self.__class__.__name__\n        if not self._is_init:\n            if self.init_cfg:\n                print_log(\n                    f'initialize {module_name} with init_cfg {self.init_cfg}',\n                    logger=logger_name,\n                    level=logging.DEBUG)\n                initialize(self, self.init_cfg)\n                if isinstance(self.init_cfg, dict):\n                    # prevent the parameters of\n                    # the pre-trained model\n                    # from being overwritten by\n                    # the `init_weights`\n                    if self.init_cfg['type'] == 'Pretrained':\n                        return\n\n            for m in self.children():\n                if hasattr(m, 'init_weights'):\n                    m.init_weights()\n                    # users may overload the `init_weights`\n                    update_init_info(\n                        m,\n                        init_info=f'Initialized by '\n                        f'user-defined `init_weights`'\n                        f' in {m.__class__.__name__} ')\n\n            self._is_init = True\n        else:\n            warnings.warn(f'init_weights of {self.__class__.__name__} has '\n                          f'been called more than once.')\n\n        if is_top_level_module:\n            # self._dump_init_info(logger_name)\n            self._dump_init_info()\n\n            for sub_module in self.modules():\n                del sub_module._params_init_info\n\n    @master_only\n    def _dump_init_info(self):\n        \"\"\"Dump the initialization information to a file named\n        `initialization.log.json` in workdir.\n\n        Args:\n            logger_name (str): The name of logger.\n        \"\"\"\n\n        logger = MMLogger.get_current_instance()\n        logger_name = logger.instance_name\n        with_file_handler = False\n        # dump the information to the logger file if there is a `FileHandler`\n        for handler in logger.handlers:\n            if isinstance(handler, FileHandler):\n                handler.stream.write(\n                    'Name of parameter - Initialization information\\n')\n                for name, param in self.named_parameters():\n                    handler.stream.write(\n                        f'\\n{name} - {param.shape}: '\n                        f\"\\n{self._params_init_info[param]['init_info']} \\n\")\n                handler.stream.flush()\n                with_file_handler = True\n        if not with_file_handler:\n            for name, param in self.named_parameters():\n                print_log(\n                    f'\\n{name} - {param.shape}: '\n                    f\"\\n{self._params_init_info[param]['init_info']} \\n \",\n                    logger=logger_name)\n\n    def __repr__(self):\n        s = super().__repr__()\n        if self.init_cfg:\n            s += f'\\ninit_cfg={self.init_cfg}'\n        return s\n\n\nclass Sequential(BaseModule, nn.Sequential):\n    \"\"\"Sequential module in openmmlab.\n\n    Ensures that all modules in ``Sequential`` have a different initialization\n    strategy than the outer model\n\n    Args:\n        init_cfg (dict, optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self, *args, init_cfg: Optional[dict] = None):\n        BaseModule.__init__(self, init_cfg)\n        nn.Sequential.__init__(self, *args)\n\n\nclass ModuleList(BaseModule, nn.ModuleList):\n    \"\"\"ModuleList in openmmlab.\n\n    Ensures that all modules in ``ModuleList`` have a different initialization\n    strategy than the outer model\n\n    Args:\n        modules (iterable, optional): An iterable of modules to add.\n        init_cfg (dict, optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self,\n                 modules: Optional[Iterable] = None,\n                 init_cfg: Optional[dict] = None):\n        BaseModule.__init__(self, init_cfg)\n        nn.ModuleList.__init__(self, modules)\n\n\nclass ModuleDict(BaseModule, nn.ModuleDict):\n    \"\"\"ModuleDict in openmmlab.\n\n    Ensures that all modules in ``ModuleDict`` have a different initialization\n    strategy than the outer model\n\n    Args:\n        modules (dict, optional): A mapping (dictionary) of (string: module)\n            or an iterable of key-value pairs of type (string, module).\n        init_cfg (dict, optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self,\n                 modules: Optional[dict] = None,\n                 init_cfg: Optional[dict] = None):\n        BaseModule.__init__(self, init_cfg)\n        nn.ModuleDict.__init__(self, modules)\n", "path": "mmengine/model/base_module.py"}], "after_files": [{"content": "# Copyright (c) OpenMMLab. All rights reserved.\nimport copy\nimport logging\nimport warnings\nfrom abc import ABCMeta\nfrom collections import defaultdict\nfrom logging import FileHandler\nfrom typing import Iterable, List, Optional, Union\n\nimport torch.nn as nn\n\nfrom mmengine.dist import master_only\nfrom mmengine.logging import MMLogger, print_log\nfrom .weight_init import initialize, update_init_info\n\n\nclass BaseModule(nn.Module, metaclass=ABCMeta):\n    \"\"\"Base module for all modules in openmmlab. ``BaseModule`` is a wrapper of\n    ``torch.nn.Module`` with additional functionality of parameter\n    initialization. Compared with ``torch.nn.Module``, ``BaseModule`` mainly\n    adds three attributes.\n\n    - ``init_cfg``: the config to control the initialization.\n    - ``init_weights``: The function of parameter initialization and recording\n      initialization information.\n    - ``_params_init_info``: Used to track the parameter initialization\n      information. This attribute only exists during executing the\n      ``init_weights``.\n\n    Note:\n        :obj:`PretrainedInit` has a higher priority than any other\n        initializer. The loaded pretrained weights will overwrite\n        the previous initialized weights.\n\n    Args:\n        init_cfg (dict or List[dict], optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self, init_cfg: Union[dict, List[dict], None] = None):\n        \"\"\"Initialize BaseModule, inherited from `torch.nn.Module`\"\"\"\n\n        # NOTE init_cfg can be defined in different levels, but init_cfg\n        # in low levels has a higher priority.\n\n        super().__init__()\n        # define default value of init_cfg instead of hard code\n        # in init_weights() function\n        self._is_init = False\n\n        self.init_cfg = copy.deepcopy(init_cfg)\n\n        # Backward compatibility in derived classes\n        # if pretrained is not None:\n        #     warnings.warn('DeprecationWarning: pretrained is a deprecated \\\n        #         key, please consider using init_cfg')\n        #     self.init_cfg = dict(type='Pretrained', checkpoint=pretrained)\n\n    @property\n    def is_init(self):\n        return self._is_init\n\n    def init_weights(self):\n        \"\"\"Initialize the weights.\"\"\"\n\n        is_top_level_module = False\n        # check if it is top-level module\n        if not hasattr(self, '_params_init_info'):\n            # The `_params_init_info` is used to record the initialization\n            # information of the parameters\n            # the key should be the obj:`nn.Parameter` of model and the value\n            # should be a dict containing\n            # - init_info (str): The string that describes the initialization.\n            # - tmp_mean_value (FloatTensor): The mean of the parameter,\n            #       which indicates whether the parameter has been modified.\n            # this attribute would be deleted after all parameters\n            # is initialized.\n            self._params_init_info = defaultdict(dict)\n            is_top_level_module = True\n\n            # Initialize the `_params_init_info`,\n            # When detecting the `tmp_mean_value` of\n            # the corresponding parameter is changed, update related\n            # initialization information\n            for name, param in self.named_parameters():\n                self._params_init_info[param][\n                    'init_info'] = f'The value is the same before and ' \\\n                                   f'after calling `init_weights` ' \\\n                                   f'of {self.__class__.__name__} '\n                self._params_init_info[param][\n                    'tmp_mean_value'] = param.data.mean().cpu()\n\n            # pass `params_init_info` to all submodules\n            # All submodules share the same `params_init_info`,\n            # so it will be updated when parameters are\n            # modified at any level of the model.\n            for sub_module in self.modules():\n                sub_module._params_init_info = self._params_init_info\n\n        logger = MMLogger.get_current_instance()\n        logger_name = logger.instance_name\n\n        module_name = self.__class__.__name__\n        if not self._is_init:\n            if self.init_cfg:\n                print_log(\n                    f'initialize {module_name} with init_cfg {self.init_cfg}',\n                    logger=logger_name,\n                    level=logging.DEBUG)\n\n                init_cfgs = self.init_cfg\n                if isinstance(self.init_cfg, dict):\n                    init_cfgs = [self.init_cfg]\n\n                # PretrainedInit has higher priority than any other init_cfg.\n                # Therefore we initialize `pretrained_cfg` last to overwrite\n                # the previous initialized weights.\n                # See details in https://github.com/open-mmlab/mmengine/issues/691 # noqa E501\n                other_cfgs = []\n                pretrained_cfg = []\n                for init_cfg in init_cfgs:\n                    assert isinstance(init_cfg, dict)\n                    if init_cfg['type'] == 'Pretrained':\n                        pretrained_cfg.append(init_cfg)\n                    else:\n                        other_cfgs.append(init_cfg)\n\n                initialize(self, other_cfgs)\n\n            for m in self.children():\n                if hasattr(m, 'init_weights'):\n                    m.init_weights()\n                    # users may overload the `init_weights`\n                    update_init_info(\n                        m,\n                        init_info=f'Initialized by '\n                        f'user-defined `init_weights`'\n                        f' in {m.__class__.__name__} ')\n            if self.init_cfg and pretrained_cfg:\n                initialize(self, pretrained_cfg)\n            self._is_init = True\n        else:\n            warnings.warn(f'init_weights of {self.__class__.__name__} has '\n                          f'been called more than once.')\n\n        if is_top_level_module:\n            # self._dump_init_info(logger_name)\n            self._dump_init_info()\n\n            for sub_module in self.modules():\n                del sub_module._params_init_info\n\n    @master_only\n    def _dump_init_info(self):\n        \"\"\"Dump the initialization information to a file named\n        `initialization.log.json` in workdir.\n\n        Args:\n            logger_name (str): The name of logger.\n        \"\"\"\n\n        logger = MMLogger.get_current_instance()\n        logger_name = logger.instance_name\n        with_file_handler = False\n        # dump the information to the logger file if there is a `FileHandler`\n        for handler in logger.handlers:\n            if isinstance(handler, FileHandler):\n                handler.stream.write(\n                    'Name of parameter - Initialization information\\n')\n                for name, param in self.named_parameters():\n                    handler.stream.write(\n                        f'\\n{name} - {param.shape}: '\n                        f\"\\n{self._params_init_info[param]['init_info']} \\n\")\n                handler.stream.flush()\n                with_file_handler = True\n        if not with_file_handler:\n            for name, param in self.named_parameters():\n                print_log(\n                    f'\\n{name} - {param.shape}: '\n                    f\"\\n{self._params_init_info[param]['init_info']} \\n \",\n                    logger=logger_name)\n\n    def __repr__(self):\n        s = super().__repr__()\n        if self.init_cfg:\n            s += f'\\ninit_cfg={self.init_cfg}'\n        return s\n\n\nclass Sequential(BaseModule, nn.Sequential):\n    \"\"\"Sequential module in openmmlab.\n\n    Ensures that all modules in ``Sequential`` have a different initialization\n    strategy than the outer model\n\n    Args:\n        init_cfg (dict, optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self, *args, init_cfg: Optional[dict] = None):\n        BaseModule.__init__(self, init_cfg)\n        nn.Sequential.__init__(self, *args)\n\n\nclass ModuleList(BaseModule, nn.ModuleList):\n    \"\"\"ModuleList in openmmlab.\n\n    Ensures that all modules in ``ModuleList`` have a different initialization\n    strategy than the outer model\n\n    Args:\n        modules (iterable, optional): An iterable of modules to add.\n        init_cfg (dict, optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self,\n                 modules: Optional[Iterable] = None,\n                 init_cfg: Optional[dict] = None):\n        BaseModule.__init__(self, init_cfg)\n        nn.ModuleList.__init__(self, modules)\n\n\nclass ModuleDict(BaseModule, nn.ModuleDict):\n    \"\"\"ModuleDict in openmmlab.\n\n    Ensures that all modules in ``ModuleDict`` have a different initialization\n    strategy than the outer model\n\n    Args:\n        modules (dict, optional): A mapping (dictionary) of (string: module)\n            or an iterable of key-value pairs of type (string, module).\n        init_cfg (dict, optional): Initialization config dict.\n    \"\"\"\n\n    def __init__(self,\n                 modules: Optional[dict] = None,\n                 init_cfg: Optional[dict] = None):\n        BaseModule.__init__(self, init_cfg)\n        nn.ModuleDict.__init__(self, modules)\n", "path": "mmengine/model/base_module.py"}]}