[bug]
run follow code:
vllm serve Qwen/Qwen2.5-7B-Instruct-1M
--tensor-parallel-size 4
--max-model-len 1010000
--enable-chunked-prefill --max-num-batched-tokens 131072
--enforce-eager
--max-num-seqs 1
bug:
INFO 03-18 14:25:35 model_runner_base.py:119] Writing input of failed execution to /tmp/err_execute_model_input_20250318-142535.pkl...
WARNING 03-18 14:25:35 model_runner_base.py:142] Failed to pickle inputs of failed execution: CUDA error: device-side assert triggered
WARNING 03-18 14:25:35 model_runner_base.py:142] CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
WARNING 03-18 14:25:35 model_runner_base.py:142] For debugging consider passing CUDA_LAUNCH_BLOCKING=1
WARNING 03-18 14:25:35 model_runner_base.py:142] Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.
WARNING 03-18 14:25:35 model_runner_base.py:142]
ERROR 03-18 14:25:35 engine.py:381] Error in model execution: CUDA error: device-side assert triggered
ERROR 03-18 14:25:35 engine.py:381] CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
ERROR 03-18 14:25:35 engine.py:381] For debugging consider passing CUDA_LAUNCH_BLOCKING=1
ERROR 03-18 14:25:35 engine.py:381] Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.
ERROR 03-18 14:25:35 engine.py:381] Traceback (most recent call last):
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner_base.py", line 115, in _wrapper
ERROR 03-18 14:25:35 engine.py:381] return func(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 1718, in execute_model
ERROR 03-18 14:25:35 engine.py:381] hidden_or_intermediate_states = model_executable(
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
ERROR 03-18 14:25:35 engine.py:381] return self._call_impl(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
ERROR 03-18 14:25:35 engine.py:381] return forward_call(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 498, in forward
ERROR 03-18 14:25:35 engine.py:381] hidden_states = self.model(input_ids, positions, kv_caches,
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/compilation/decorators.py", line 170, in call
ERROR 03-18 14:25:35 engine.py:381] return self.forward(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 360, in forward
ERROR 03-18 14:25:35 engine.py:381] hidden_states, residual = layer(
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
ERROR 03-18 14:25:35 engine.py:381] return self._call_impl(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
ERROR 03-18 14:25:35 engine.py:381] return forward_call(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 267, in forward
ERROR 03-18 14:25:35 engine.py:381] hidden_states = self.self_attn(
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
ERROR 03-18 14:25:35 engine.py:381] return self._call_impl(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
ERROR 03-18 14:25:35 engine.py:381] return forward_call(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 186, in forward
ERROR 03-18 14:25:35 engine.py:381] rotary_embedded = self.rotary_emb(positions, q, k)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
ERROR 03-18 14:25:35 engine.py:381] return self._call_impl(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
ERROR 03-18 14:25:35 engine.py:381] return forward_call(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding.py", line 1053, in forward
ERROR 03-18 14:25:35 engine.py:381] key = self._apply_rotary_embedding(
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding.py", line 1086, in _apply_rotary_embedding
ERROR 03-18 14:25:35 engine.py:381] hidden_rot = hidden_rot * cos + rotate_fn(hidden_rot) * sin
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding.py", line 35, in _rotate_neox
ERROR 03-18 14:25:35 engine.py:381] return torch.cat((-x2, x1), dim=-1)
ERROR 03-18 14:25:35 engine.py:381] RuntimeError: CUDA error: device-side assert triggered
ERROR 03-18 14:25:35 engine.py:381] CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
ERROR 03-18 14:25:35 engine.py:381] For debugging consider passing CUDA_LAUNCH_BLOCKING=1
ERROR 03-18 14:25:35 engine.py:381] Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.
ERROR 03-18 14:25:35 engine.py:381]
ERROR 03-18 14:25:35 engine.py:381]
ERROR 03-18 14:25:35 engine.py:381] The above exception was the direct cause of the following exception:
ERROR 03-18 14:25:35 engine.py:381]
ERROR 03-18 14:25:35 engine.py:381] Traceback (most recent call last):
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 372, in run_mp_engine
ERROR 03-18 14:25:35 engine.py:381] engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 120, in from_engine_args
ERROR 03-18 14:25:35 engine.py:381] return cls(ipc_path=ipc_path,
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 72, in init
ERROR 03-18 14:25:35 engine.py:381] self.engine = LLMEngine(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 274, in init
ERROR 03-18 14:25:35 engine.py:381] self._initialize_kv_caches()
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 414, in _initialize_kv_caches
ERROR 03-18 14:25:35 engine.py:381] self.model_executor.determine_num_available_blocks())
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 77, in determine_num_available_blocks
ERROR 03-18 14:25:35 engine.py:381] results = self.collective_rpc("determine_num_available_blocks")
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 258, in collective_rpc
ERROR 03-18 14:25:35 engine.py:381] return self._run_workers(method, *args, **(kwargs or {}))
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/executor/mp_distributed_executor.py", line 183, in _run_workers
ERROR 03-18 14:25:35 engine.py:381] driver_worker_output = run_method(self.driver_worker, sent_method,
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/utils.py", line 2209, in run_method
ERROR 03-18 14:25:35 engine.py:381] return func(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
ERROR 03-18 14:25:35 engine.py:381] return func(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/worker.py", line 200, in determine_num_available_blocks
ERROR 03-18 14:25:35 engine.py:381] self.model_runner.profile_run()
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
ERROR 03-18 14:25:35 engine.py:381] return func(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 1350, in profile_run
ERROR 03-18 14:25:35 engine.py:381] self.execute_model(model_input, kv_caches, intermediate_tensors)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
ERROR 03-18 14:25:35 engine.py:381] return func(*args, **kwargs)
ERROR 03-18 14:25:35 engine.py:381] File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner_base.py", line 145, in _wrapper
ERROR 03-18 14:25:35 engine.py:381] raise type(err)(f"Error in model execution: "
ERROR 03-18 14:25:35 engine.py:381] RuntimeError: Error in model execution: CUDA error: device-side assert triggered
ERROR 03-18 14:25:35 engine.py:381] CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
ERROR 03-18 14:25:35 engine.py:381] For debugging consider passing CUDA_LAUNCH_BLOCKING=1
ERROR 03-18 14:25:35 engine.py:381] Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.
ERROR 03-18 14:25:35 engine.py:381]
ERROR 03-18 14:25:36 multiproc_worker_utils.py:122] Worker VllmWorkerProcess pid 1126236 died, exit code: -15
INFO 03-18 14:25:36 multiproc_worker_utils.py:126] Killing local vLLM worker processes
Process SpawnProcess-1:
Traceback (most recent call last):
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner_base.py", line 115, in _wrapper
return func(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 1718, in execute_model
hidden_or_intermediate_states = model_executable(
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 498, in forward
hidden_states = self.model(input_ids, positions, kv_caches,
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/compilation/decorators.py", line 170, in call
return self.forward(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 360, in forward
hidden_states, residual = layer(
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 267, in forward
hidden_states = self.self_attn(
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/models/qwen2.py", line 186, in forward
rotary_embedded = self.rotary_emb(positions, q, k)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding.py", line 1053, in forward
key = self._apply_rotary_embedding(
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding.py", line 1086, in _apply_rotary_embedding
hidden_rot = hidden_rot * cos + rotate_fn(hidden_rot) * sin
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/model_executor/layers/rotary_embedding.py", line 35, in _rotate_neox
return torch.cat((-x2, x1), dim=-1)
RuntimeError: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 383, in run_mp_engine
raise e
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 372, in run_mp_engine
engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 120, in from_engine_args
return cls(ipc_path=ipc_path,
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/engine.py", line 72, in init
self.engine = LLMEngine(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 274, in init
self._initialize_kv_caches()
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 414, in _initialize_kv_caches
self.model_executor.determine_num_available_blocks())
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 77, in determine_num_available_blocks
results = self.collective_rpc("determine_num_available_blocks")
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 258, in collective_rpc
return self._run_workers(method, *args, **(kwargs or {}))
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/executor/mp_distributed_executor.py", line 183, in _run_workers
driver_worker_output = run_method(self.driver_worker, sent_method,
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/utils.py", line 2209, in run_method
return func(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/worker.py", line 200, in determine_num_available_blocks
self.model_runner.profile_run()
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 1350, in profile_run
self.execute_model(model_input, kv_caches, intermediate_tensors)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/worker/model_runner_base.py", line 145, in _wrapper
raise type(err)(f"Error in model execution: "
RuntimeError: Error in model execution: CUDA error: device-side assert triggered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.
[rank0]:[W318 14:25:36.878856386 ProcessGroupNCCL.cpp:1250] Warning: WARNING: process group has NOT been destroyed before we destruct ProcessGroupNCCL. On normal program exit, the application should call destroy_process_group to ensure that any pending NCCL operations have finished in this process. In rare cases this process can exit before this point and block the progress of another member of the process group. This constraint has always been present, but this warning has only been added since PyTorch 2.4 (function operator())
Task exception was never retrieved
future: <Task finished name='Task-2' coro=<MQLLMEngineClient.run_output_handler_loop() done, defined at /home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/client.py:180> exception=ZMQError('Operation not supported')>
Traceback (most recent call last):
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/engine/multiprocessing/client.py", line 186, in run_output_handler_loop
while await self.output_socket.poll(timeout=VLLM_RPC_TIMEOUT
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/zmq/_future.py", line 372, in poll
raise _zmq.ZMQError(_zmq.ENOTSUP)
zmq.error.ZMQError: Operation not supported
Traceback (most recent call last):
File "/home/mllabs/anaconda3/envs/oyby/bin/vllm", line 8, in
sys.exit(main())
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/scripts.py", line 201, in main
args.dispatch_function(args)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/scripts.py", line 42, in serve
uvloop.run(run_server(args))
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/uvloop/init.py", line 82, in run
return loop.run_until_complete(wrapper())
File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/uvloop/init.py", line 61, in wrapper
return await main
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 796, in run_server
async with build_async_engine_client(args) as engine_client:
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/contextlib.py", line 199, in aenter
return await anext(self.gen)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 125, in build_async_engine_client
async with build_async_engine_client_from_engine_args(
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/contextlib.py", line 199, in aenter
return await anext(self.gen)
File "/home/mllabs/anaconda3/envs/oyby/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 219, in build_async_engine_client_from_engine_args
raise RuntimeError(
RuntimeError: Engine process failed to start. See stack trace for the root cause.
/home/mllabs/anaconda3/envs/oyby/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown
warnings.warn('resource_tracker: There appear to be %d '
/home/mllabs/anaconda3/envs/oyby/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown
warnings.warn('resource_tracker: There appear to be %d '