loss growing to negative infinity val accuracy remaining constant, testing accuracy above 100%
@ripjohnbrown1859 Could you please have a look at this gist and confirm the error reported [here](https://colab.research.google.com/gist/sushreebarsa/2f66623bf38d27b0a0a8b87bc9c56cc4/65468.ipynb#scrollTo=vtO_j7dCzOyO). 
Thank you! that is because I am running 2 gpus and the notebook does not have access to 2 gpus so it says the list index for gpus is out of range. it should work if you simply cut out all the gpu[1] stuff. this is not a gpu issue, the issue persists whether or not i run it on my gpus. or cpu. 
 Hi @ripjohnbrown1859 ,

Since the submitted code snippet has dependencies with your dataset could you please submit a minimal code with some dummy set. There seems to be issue as I can notice accuracy is >1 from the attached logs. heres the link to the dataset. https://physionet.org/content/ecg-arrhythmia/1.0.0/
### Issue type

Bug

### Have you reproduced the bug with TensorFlow Nightly?

No

### Source

binary

### TensorFlow version

tf 2.16.1

### Custom code

Yes

### OS platform and distribution

wsl ubuntu 22.04

### Mobile device

_No response_

### Python version

3.10

### Bazel version

_No response_

### GCC/compiler version

_No response_

### CUDA/cuDNN version

8.6.0

### GPU model and memory

sli titan x maxwell

### Current behavior?

currently my loss is increasing to negative infinity, my test accuracy is sitting around 1.85 and my val accuracy is staying at .9374 no matter how may epochs i run. i am training my model to read ekgs, i have 64 not mutually exclusive outputs that correspond with different diagnoses. each input is a 12 lead ekg with 5000 time series datapoints for each lead.


### Standalone code to reproduce the issue

```shell
tensorflow as tf
import os
import concurrent.futures
import csv
import numpy as np
import scipy.io as sio
from tensorflow.keras.callbacks import (ModelCheckpoint, TensorBoard, ReduceLROnPlateau,
                                        CSVLogger, EarlyStopping)
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.utils import Sequence
from tensorflow.keras.layers import (
    Input, Conv1D, MaxPooling1D, Dropout, BatchNormalization, Activation, Add, Flatten, Dense)
from tensorflow.keras.models import Model
import numpy as np


class ResidualUnit(object):

    def __init__(self, n_samples_out, n_filters_out, kernel_initializer='he_normal',
                 dropout_keep_prob=0.8, kernel_size=20, preactivation=True,
                 postactivation_bn=False, activation_function='relu'):
        self.n_samples_out = n_samples_out
        self.n_filters_out = n_filters_out
        self.kernel_initializer = kernel_initializer
        self.dropout_rate = 1 - dropout_keep_prob
        self.kernel_size = kernel_size
        self.preactivation = preactivation
        self.postactivation_bn = postactivation_bn
        self.activation_function = activation_function

    def _skip_connection(self, y, downsample, n_filters_in):
        """Implement skip connection."""
        # Deal with downsampling
        if downsample > 1:
            y = MaxPooling1D(downsample, strides=downsample, padding='same')(y)
        elif downsample == 1:
            y = y
        else:
            raise ValueError("Number of samples should always decrease.")
        # Deal with n_filters dimension increase
        if n_filters_in != self.n_filters_out:
            # This is one of the two alternatives presented in ResNet paper
            # Other option is to just fill the matrix with zeros.
            y = Conv1D(self.n_filters_out, 1, padding='same',
                       use_bias=False, kernel_initializer=self.kernel_initializer)(y)
        return y

    def _batch_norm_plus_activation(self, x):
        if self.postactivation_bn:
            x = Activation(self.activation_function)(x)
            x = BatchNormalization(center=False, scale=False)(x)
        else:
            x = BatchNormalization()(x)
            x = Activation(self.activation_function)(x)
        return x

    def __call__(self, inputs):
        """Residual unit."""
        x, y = inputs
        n_samples_in = y.shape[1]
        downsample = n_samples_in // self.n_samples_out
        n_filters_in = y.shape[2]
        y = self._skip_connection(y, downsample, n_filters_in)
        # 1st layer
        x = Conv1D(self.n_filters_out, self.kernel_size, padding='same',
                   use_bias=False, kernel_initializer=self.kernel_initializer)(x)
        x = self._batch_norm_plus_activation(x)
        if self.dropout_rate > 0:
            x = Dropout(self.dropout_rate)(x)

        # 2nd layer
        x = Conv1D(self.n_filters_out, self.kernel_size, strides=downsample,
                   padding='same', use_bias=False,
                   kernel_initializer=self.kernel_initializer)(x)
        if self.preactivation:
            x = Add()([x, y])  # Sum skip connection and main connection
            y = x
            x = self._batch_norm_plus_activation(x)
            if self.dropout_rate > 0:
                x = Dropout(self.dropout_rate)(x)
        else:
            x = BatchNormalization()(x)
            x = Add()([x, y])  # Sum skip connection and main connection
            x = Activation(self.activation_function)(x)
            if self.dropout_rate > 0:
                x = Dropout(self.dropout_rate)(x)
            y = x
        return [x, y]


def get_model(n_classes, last_layer='sigmoid'):
    kernel_size = 20
    kernel_initializer = 'he_normal'
    signal = Input(shape=(5000, 12), dtype=tf.float32, name='signal')
    x = signal
    x = Conv1D(64, kernel_size, padding='same', use_bias=False,
               kernel_initializer=kernel_initializer)(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x, y = ResidualUnit(1024, 128, kernel_size=kernel_size, 
                        kernel_initializer=kernel_initializer)([x, x])
    x, y = ResidualUnit(256, 196, kernel_size=kernel_size,
                        kernel_initializer=kernel_initializer)([x, y])
    x, y = ResidualUnit(64, 256, kernel_size=kernel_size,
                        kernel_initializer=kernel_initializer)([x, y])
    x, _ = ResidualUnit(16, 320, kernel_size=kernel_size,
                        kernel_initializer=kernel_initializer)([x, y])
    x = Flatten()(x)
    diagn = Dense(n_classes, activation=last_layer, kernel_initializer=kernel_initializer)(x)
    model = Model(signal, diagn)
    return model
gpu = tf.config.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(gpu[0], True)
tf.config.experimental.set_memory_growth(gpu[1], True)
#tf.config.experimental.set_virtual_device_configuration(gpu[1], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=11000)])
#tf.config.experimental.set_virtual_device_configuration(gpu[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=11000)])
print(tf.version.GIT_VERSION, tf.version.VERSION)
print(tf.config.list_logical_devices())
num_classes=64
num_samples = 45561
options = tf.data.Options()
options.autotune.ram_budget = 51539607552
#from tensorflow.keras import layers, models
def parse_hea_file(file_path):
    with open(file_path, 'r') as file:
        lines = file.readlines()
        dx_values = None
        for line in lines:
            if line.startswith('#Dx'):
                dx_values = line.strip().split(': ')[1].split(',') if len(line.strip().split(': ')) > 1 else 0
                dx_values = [dx[key] for key in dx_values if key in dx]
                break
    return dx_values
# Initialize an empty dictionary
dx = {}

# Open the CSV file for reading
def create_dx_dict():
    with open('a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0/ConditionNames_SNOMED-CT.csv', newline='') as csvfile:
    # Create a CSV reader object
        csv_reader = csv.reader(csvfile)
    
    # Skip the header row
        next(csv_reader)
        i = 0
    # Iterate over each row in the CSV file
        for row in csv_reader:
            i += 1
        # Assuming the CSV file has 3 columns and they are indexed as 0, 1, and 2
        # Construct the dictionary with the third column as keys and the second column as values
            key = row[2]
            value = i
            dx[key] = value
    return dx
dx=create_dx_dict()

# Specify the path to the parent directory
def normalize_data(data):
    for i in range(data.shape[1]):
        for j in range(data.shape[0]):
            max_abs = 32768
            data[j,i] /= max_abs
    return data
input_dataset = []
output_dataset = []
# Get a list of directories within the parent directory
def process_subdirectory(subdir_path):
    output_data = []
    input_data = []
    sub_subdirectories = [d for d in os.listdir(subdir_path) if os.path.isdir(os.path.join(subdir_path, d))]
    for sub_subdir in sub_subdirectories:
        files_path = os.path.join(subdir_path, sub_subdir)
        print(files_path)
        files = [f for f in os.listdir(files_path) if os.path.isfile(os.path.join(files_path, f))]
        for file in files:
            if file.endswith('.hea'):
                dx_values = parse_hea_file(os.path.join(files_path, file))
                output_data.append(dx_values)
                matfile = os.path.splitext(file)[0] + '.mat'
                data = sio.loadmat(os.path.join(files_path, matfile))
                ecg_data = data['val']
                input_data.append(np.array(ecg_data))
    return input_data, output_data
class ECGDataGenerator(Sequence):
    def __init__(self, data_dir ="a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0/WFDBRecords", batch_size=512):
        self.data_dir = data_dir
        self.batch_size = batch_size
        self.filenames = self._find_data_files()
    def __len__(self):
        return int(np.ceil(len(self.filenames) / self.batch_size))
    def _find_data_files(self):
        filenames = []
        subdirectories = [d for d in os.listdir(self.data_dir) if os.path.isdir(os.path.join(self.data_dir, d))]
        for subdir in subdirectories:
            subsubdirectories = [d for d in os.listdir(os.path.join(self.data_dir, subdir)) if os.path.isdir(os.path.join(self.data_dir, subdir, d))]
            for subsubdir in subsubdirectories:
                files_path = os.path.join(self.data_dir, subdir, subsubdir)
                files = [f for f in os.listdir(files_path) if os.path.isfile(os.path.join(files_path, f))]
                for file in files:
                    if file.endswith('.hea'): 
                        file = os.path.join(subdir, subsubdir, file)
                        filenames.append(file)
        return filenames
    def __getitem__(self, idx):
        batch_filenames = self.filenames[idx * self.batch_size:(idx + 1) * self.batch_size]
        with concurrent.futures.ProcessPoolExecutor() as executor:
            batch_data_labels = list(executor.map(self._process_file, batch_filenames))
        batch_data, batch_labels = zip(*batch_data_labels)
        return np.array(batch_data), np.array(batch_labels)

    def _process_file(self, filename):
        dx_values = parse_hea_file(os.path.join(self.data_dir, filename))
        padded_dx_values = pad_sequences([dx_values], maxlen=64, padding = "post")[0]
        if dx_values is not None:
            mat_filename = os.path.splitext(filename)[0] + '.mat'
            mat_data = sio.loadmat(os.path.join(self.data_dir, mat_filename)).get('val')
            mat_data = np.reshape(mat_data, (5000,12))
            normalized_data = normalize_data(mat_data)
            
            return normalized_data, padded_dx_values
lr = 0.001
strategy = tf.distribute.MirroredStrategy()
callbacks = [ReduceLROnPlateau(monitor='val_loss',
                                   factor=0.1,
                                   patience=7,
                                   min_lr=lr / 100),
                 EarlyStopping(patience=9,  # Patience should be larger than the one in ReduceLROnPlateau
                               min_delta=0.00001)]
with tf.device('/CPU:0'):
    dataset = tf.data.Dataset.from_generator(generator=ECGDataGenerator, output_types=(tf.float32, tf.float32), output_shapes=([None, 5000, 12], [None, 64]))
    dataset = dataset.shuffle(86)
    training_dataset = dataset.take(80)
    validation_dataset = dataset.skip(80)

with strategy.scope():
    training_dataset = training_dataset.cache().repeat().prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
    validation_dataset = validation_dataset.cache().repeat().prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
    model = get_model(num_classes)
    print(model.summary())
    model.compile(optimizer=tf.keras.optimizers.Adam(lr), loss=tf.keras.losses.BinaryCrossentropy(), metrics=['binary_accuracy'])
model.fit(training_dataset, validation_data=validation_dataset, epochs=70, steps_per_epoch=80, validation_steps=7, callbacks=callbacks)
model.export("ecg_model", "tf_saved_model")
```


### Relevant log output

```shell
80/80 ━━━━━━━━━━━━━━━━━━━━ 2449s 16s/step - binary_accuracy: 1.8140 - loss: -92.5793 - val_binary_accuracy: 0.9372 - val_loss: -96.8444 - learning_rate: 0.0010
Epoch 2/70
80/80 ━━━━━━━━━━━━━━━━━━━━ 110s 1s/step - binary_accuracy: 1.8469 - loss: -742.3646 - val_binary_accuracy: 0.9374 - val_loss: -279.6367 - learning_rate: 0.0010
Epoch 3/70
80/80 ━━━━━━━━━━━━━━━━━━━━ 111s 1s/step - binary_accuracy: 1.8468 - loss: -1657.0448 - val_binary_accuracy: 0.9374 - val_loss: -712.0684 - learning_rate: 0.0010
Epoch 4/70
80/80 ━━━━━━━━━━━━━━━━━━━━ 111s 1s/step - binary_accuracy: 1.8459 - loss: -2816.5115 - val_binary_accuracy: 0.9374 - val_loss: -1348.7988 - learning_rate: 0.0010
etc...
```
