TF throws: "'visible_device_list' listed an invalid Device id" when using non-GPU PluggableDevices
Hi @mergian ,

Thanks for reporting. I think reproduction of this issue needs non-NVIDIA GPUs,If I am not wrong. I am assuming that you have non-GPU pluggable devices which are actually identified as GPU by TF which is the problem,right?

I did refer some resources [gpu-plugins ](https://www.tensorflow.org/install/gpu_plugins) and [pluggable-device-for-tensorflow.md](https://github.com/tensorflow/community/blob/master/rfcs/20200624-pluggable-device-for-tensorflow.md).As per my understanding from here if the pluggable device registered as GPU then it shall be added  to GPU list and implementation done accordingly. 

As per my understanding there might be non GPU pluggable devices also which can be usable by TF. At least this is my understanding from the `.readme` file attached earlier in this comment and correct me if I am wrong. In Your proposed code all the devices which are not registered as GPU will be ignored right.I think in that case we need to check for more. Please share your thoughts . Dear @SuryanarayanaY 

yes you are correct.

Our PluggableDevice plugin adds non-GPU devices. The code that I marked however assumes all PluggableDevices to be GPUs, therefore the `visible_devices` list will be populated with invalid ids, which then later cause errors when being checked internally.

My proposal is, that for this particular code pice, that looks for TensorFlow-Native-GPUs and PluggableDevice-GPUs and combines them, to check if the `.device_type` is really `GPU`.

If needed it should be possible to create a fake PluggableDevice-Plugin, that creates non existing devices. Then you could reproduce this error even without having our devices available. But I think could take some hours to do so. Hi @mergian ,

Thanks for confirmation. With your proposal only PluggableDevice-GPUs can be detectable y TF. I doubt whether this implementation only limited to GPUs as .[readme](https://github.com/tensorflow/community/blob/master/rfcs/20200624-pluggable-device-for-tensorflow.md) talks about more than GPU type devices and I may be wrong also here. I would like to hear from dev team on this.But I feel there is some cases not covered in implementation.

Thanks!

 No. My bugfix aims at supporting also other PluggableDevice types. Currently, when you mix GPUs and Non-GPUs, you'll get the above mentioned error, as within this GPU specific initialization function, it's not properly distinguishing between GPUs and other device types, and therefore initializes this "gpu_options" data structure with invalid values.

I build a demonstrator for this. Here is a fake PluggableDevice, that creates 2 "NO_GPU" devices.

```cpp
#include <tensorflow/c/experimental/stream_executor/stream_executor.h>

static void get_device_count(const SP_Platform* platform, int* device_count, TF_Status* status) {
	*device_count = 2;
	TF_SetStatus(status, TF_OK,	"");
}

static void create_device(const SP_Platform* platform, SE_CreateDeviceParams* params, TF_Status* status) {
	params->device->device_handle	= 0;
	params->device->hardware_name	= "TEST DEVICE";
	params->device->device_vendor	= "SOME VENDOR";
	params->device->pci_bus_id		= "0000:00:00.0";
	TF_SetStatus(status, TF_OK,	"");
}

static int32_t get_numa_node(const SP_Device* device) {
	return device->ordinal;
}

static int64_t get_memory_bandwidth(const SP_Device* device) {
	return -1;
}

static double get_gflops(const SP_Device* device) {
	return -1;
}

static void create_device_fns(const SP_Platform* platform, SE_CreateDeviceFnsParams* params, TF_Status* status) {
	params->device_fns->get_gflops				= &get_gflops;
	params->device_fns->get_memory_bandwidth	= &get_memory_bandwidth;
	params->device_fns->get_numa_node			= &get_numa_node;
	TF_SetStatus(status, TF_OK,	"");
}

static uint64_t nanoseconds(SP_Timer timer) {
	return 0;
}

static void destroy_device(const SP_Platform* platform, SP_Device* device) {
}

static void destroy_device_fns(const SP_Platform* platform, SP_DeviceFns* device_fns) {
}

static void destroy_platform(SP_Platform* platform) {
}

static void destroy_platform_fns(SP_PlatformFns* platform_fns) {
}

static void destroy_stream_executor(const SP_Platform* platform, SP_StreamExecutor* stream_executor) {
}

static void destroy_timer_fns(const SP_Platform* platform, SP_TimerFns* timer_fns) {
}

static void destroy_event(const SP_Device* device, SP_Event event) {
}

static void record_event_helper(SP_Event event) {
}

static void create_event(const SP_Device* device, SP_Event* event, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void create_timer_fns(const SP_Platform* platform, SP_TimerFns* timer, TF_Status* status) {
	timer->nanoseconds = &nanoseconds;
	TF_SetStatus(status, TF_OK,	"");
}

static SE_EventStatus get_event_status(const SP_Device* device, SP_Event event) {	
	return SE_EventStatus();
}

static void record_event(const SP_Device* device, SP_Stream stream, SP_Event event, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void wait_for_event(const SP_Device* const device, SP_Stream stream, SP_Event event, TF_Status* const status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void allocate(const SP_Device* device, uint64_t size, int64_t memory_space, SP_DeviceMemoryBase* mem) {
}

static void deallocate(const SP_Device* device, SP_DeviceMemoryBase* memory) {
}

static void* host_memory_allocate(const SP_Device* device, uint64_t size) {
	return 0;
}

static void host_memory_deallocate(const SP_Device* device, void* ptr) {
}

static TF_Bool get_allocator_stats(const SP_Device* device, SP_AllocatorStats* stats) {
	return 1;
}

static TF_Bool device_memory_usage(const SP_Device* device, int64_t* free, int64_t* total) {
	return 1;
}

static void create_stream(const SP_Device* device, SP_Stream* stream, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void destroy_stream(const SP_Device* device, SP_Stream stream) {
}

static void create_stream_dependency(const SP_Device* device, SP_Stream dependent, SP_Stream other, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void get_stream_status(const SP_Device* device, SP_Stream stream, TF_Status* status) {
}

static void memcpy_dtoh(const SP_Device* device, SP_Stream stream, void* host_dst, const SP_DeviceMemoryBase* device_src, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void memcpy_htod(const SP_Device* device, SP_Stream stream, SP_DeviceMemoryBase* device_dst, const void* host_src, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void memcpy_dtod(const SP_Device* device, SP_Stream stream, SP_DeviceMemoryBase* device_dst, const SP_DeviceMemoryBase* device_src, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void sync_memcpy_dtoh(const SP_Device* device, void* host_dst, const SP_DeviceMemoryBase* device_src, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void sync_memcpy_htod(const SP_Device* device, SP_DeviceMemoryBase* device_dst, const void* host_src, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void sync_memcpy_dtod(const SP_Device* device, SP_DeviceMemoryBase* device_dst, const SP_DeviceMemoryBase* device_src, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void block_host_for_event(const SP_Device* device, SP_Event event, TF_Status* status) {
}

static void block_host_until_done(const SP_Device* device, SP_Stream stream, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void synchronize_all_activity(const SP_Device* device, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static TF_Bool host_callback(const SP_Device* device, SP_Stream stream, SE_StatusCallbackFn callback_fn, void* callback_arg) {
	return true;
}

static void create_timer(const SP_Device* device, SP_Timer* timer, TF_Status* status) {
}

static void destroy_timer(const SP_Device* device, SP_Timer timer) {
}

static void start_timer(const SP_Device* device, SP_Stream stream, SP_Timer timer, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void stop_timer(const SP_Device* device, SP_Stream stream, SP_Timer timer, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void mem_zero(const SP_Device* device, SP_Stream stream, SP_DeviceMemoryBase* location, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void memset8(const SP_Device* device, SP_Stream stream, SP_DeviceMemoryBase* location, uint8_t pattern, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

static void memset32(const SP_Device* device, SP_Stream stream, SP_DeviceMemoryBase* location, uint32_t pattern, uint64_t size, TF_Status* status) {
	TF_SetStatus(status, TF_OK,	"");
}

void create_stream_executor(const SP_Platform* platform, SE_CreateStreamExecutorParams* params, TF_Status* status) {
	params->stream_executor->allocate					= &allocate;
	params->stream_executor->block_host_for_event		= &block_host_for_event;
	params->stream_executor->block_host_until_done		= &block_host_until_done;
	params->stream_executor->create_event				= &create_event;
	params->stream_executor->create_stream				= &create_stream;
	params->stream_executor->create_stream_dependency	= &create_stream_dependency;
	params->stream_executor->create_timer				= &create_timer;
	params->stream_executor->deallocate					= &deallocate;
	params->stream_executor->destroy_event				= &destroy_event;
	params->stream_executor->destroy_stream				= &destroy_stream;
	params->stream_executor->destroy_timer				= &destroy_timer;
	params->stream_executor->device_memory_usage		= &device_memory_usage;
	params->stream_executor->get_allocator_stats		= &get_allocator_stats;
	params->stream_executor->get_event_status			= &get_event_status;
	params->stream_executor->get_stream_status			= &get_stream_status;
	params->stream_executor->host_callback				= &host_callback;
	params->stream_executor->host_memory_allocate		= &host_memory_allocate;
	params->stream_executor->host_memory_deallocate		= &host_memory_deallocate;
	params->stream_executor->memcpy_dtod				= &memcpy_dtod;
	params->stream_executor->memcpy_dtoh				= &memcpy_dtoh;
	params->stream_executor->memcpy_htod				= &memcpy_htod;
	params->stream_executor->record_event				= &record_event;
	params->stream_executor->start_timer				= &start_timer;
	params->stream_executor->stop_timer					= &stop_timer;
	params->stream_executor->sync_memcpy_dtod			= &sync_memcpy_dtod;
	params->stream_executor->sync_memcpy_dtoh			= &sync_memcpy_dtoh;
	params->stream_executor->sync_memcpy_htod			= &sync_memcpy_htod;
	params->stream_executor->synchronize_all_activity	= &synchronize_all_activity;
	params->stream_executor->wait_for_event				= &wait_for_event;
	params->stream_executor->mem_zero					= &mem_zero;
	params->stream_executor->memset						= &memset8;
	params->stream_executor->memset32					= &memset32;
	TF_SetStatus(status, TF_OK,	"");
}

extern "C" void SE_InitPlugin(SE_PlatformRegistrationParams* params, TF_Status* status) {
	params->destroy_platform						= &destroy_platform;
	params->destroy_platform_fns					= &destroy_platform_fns;

	params->platform->name							= "SOME_PLATFORM_NAME";
	params->platform->type							= "NO_GPU";
	params->platform->supports_unified_memory		= 0;
	params->platform->use_bfc_allocator				= 0;
	params->platform->force_memory_growth			= 0;

	params->platform_fns->create_device				= &create_device;
	params->platform_fns->create_device_fns			= &create_device_fns;
	params->platform_fns->create_stream_executor	= &create_stream_executor;
	params->platform_fns->create_timer_fns			= &create_timer_fns;
	params->platform_fns->destroy_device			= &destroy_device;
	params->platform_fns->destroy_device_fns		= &destroy_device_fns;
	params->platform_fns->destroy_stream_executor	= &destroy_stream_executor;
	params->platform_fns->destroy_timer_fns			= &destroy_timer_fns;
	params->platform_fns->get_device_count			= &get_device_count;
}

extern "C" void TF_InitKernel(void) {
}
```

You can trigger the compilation + error using this:
```bash
#!/bin/bash

python_path=$(python3 -m site --user-site)

g++ -I $python_path/tensorflow/include -fpic -c pluggable.cpp -o pluggable.o
g++ -shared -L $python_path/tensorflow/ -l:libtensorflow_framework.so.2 pluggable.o -o $python_path/tensorflow-plugins/liberror.so

python3 -c "import tensorflow as tf; print('', *tf.config.list_physical_devices(), '', sep='\n'); tf.device('/GPU:0')"
```

I get the output:
```
2023-06-19 09:15:25.980567: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-19 09:15:27.181142: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT

PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')
PhysicalDevice(name='/physical_device:NO_GPU:0', device_type='NO_GPU')
PhysicalDevice(name='/physical_device:NO_GPU:1', device_type='NO_GPU')
PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')

Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 5577, in device_v2
    return device(device_name)
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 5526, in device
    return context.device(device_name_or_function)
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/eager/context.py", line 2348, in device
    ensure_initialized()
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/eager/context.py", line 2143, in ensure_initialized
    context().ensure_initialized()
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/eager/context.py", line 583, in ensure_initialized
    context_handle = pywrap_tfe.TFE_NewContext(opts)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 'visible_device_list' listed an invalid Device id '2' but visible device count is 2
```

With my proposed bugfix, the GPU device correctly gets selected:
```
2023-06-19 09:17:32.876242: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-19 09:17:36.355710: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT

PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')
PhysicalDevice(name='/physical_device:NO_GPU:0', device_type='NO_GPU')
PhysicalDevice(name='/physical_device:NO_GPU:1', device_type='NO_GPU')
PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')

2023-06-19 09:17:38.938987: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform NO_GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.
2023-06-19 09:17:38.939062: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:NO_GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: SOME_PLATFORM_NAME, pci bus id: <undefined>)
2023-06-19 09:17:39.512542: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1635] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 7371 MB memory:  -> device: 0, name: Quadro P4000, pci bus id: 0000:65:00.0, compute capability: 6.1
```

<details><summary>Click to expand!</summary> 
 
 ### Issue Type

Bug

### Have you reproduced the bug with TF nightly?

No

### Source

source

### Tensorflow Version

2.12.0

### Custom Code

Yes

### OS Platform and Distribution

Linux CentOS

### Mobile device

_No response_

### Python version

_No response_

### Bazel version

_No response_

### GCC/Compiler version

_No response_

### CUDA/cuDNN version

11.8

### GPU model and memory

_No response_

### Current Behaviour?

When using PluggableDevice API together with GPU devices, TF crashes with `tensorflow.python.framework.errors_impl.InvalidArgumentError: 'visible_device_list' listed an invalid Device id '2' but visible device count is 2` when calling `tf.device('/GPU:0')`

**For reproducing the error, it is necessary to have a PluggableDevice Plugin loaded and to have GPUs within the same system!!!**

Here the list of devices within my system:
```python3
# tf.config.list_physical_devices()
[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:VE:0', device_type='VE'), PhysicalDevice(name='/physical_device:VE:1', device_type='VE'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
```

I traced down the error to be thrown here. It gets thrown in
https://github.com/tensorflow/tensorflow/blob/e32f5b90ec16e88b23be8a5189e52ea9a420e999/tensorflow/tsl/framework/device_id_utils.cc#L46

However, it is caused by wrong values stored in the gpu_options, which get initialized here:
https://github.com/tensorflow/tensorflow/blob/0db597d0d758aba578783b5bf46c889700a45085/tensorflow/python/eager/context.py#L1206

The list of gpu_devices and ALL pluggable_devices get combined, even if they are not of the same device_type. So the list of `compatible_devices` will be:
```
[
	PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'),
	PhysicalDevice(name='/physical_device:VE:0', device_type='VE'),
	PhysicalDevice(name='/physical_device:VE:1', device_type='VE')
]
```

This causes the `visible_device_list` to be `['0', '1', '2']`, which contains invalid GPU device indices. These then get passed to `ParseVisibleDeviceList`, which throws this error.

To fix this error, it suffices to change this line:
https://github.com/tensorflow/tensorflow/blob/0db597d0d758aba578783b5bf46c889700a45085/tensorflow/python/eager/context.py#L1216

and replace it to:
```python
if dev not in gpu_devices and dev.device_type == "GPU":
```

This way, the list of `compatible_devices` will only populated with other GPUs, not with any other device types.

### Standalone code to reproduce the issue

```shell
import tensorflow as tf
print(*tf.config.list_physical_devices(), sep='\n')
tf.device('/GPU:0')
```


### Relevant log output

```shell
PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')
PhysicalDevice(name='/physical_device:VE:0', device_type='VE')
PhysicalDevice(name='/physical_device:VE:1', device_type='VE')
PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 5577, in device_v2
    return device(device_name)
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 5526, in device
    return context.device(device_name_or_function)
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/eager/context.py", line 2348, in device
    ensure_initialized()
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/eager/context.py", line 2143, in ensure_initialized
    context().ensure_initialized()
  File "/.../.local/lib/python3.8/site-packages/tensorflow/python/eager/context.py", line 583, in ensure_initialized
    context_handle = pywrap_tfe.TFE_NewContext(opts)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 'visible_device_list' listed an invalid Device id '2' but visible device count is 2
```
</details>