Tensorflow error "W tensorflow/core/data/root_dataset.cc:163] Optimization loop failed: CANCELLED: Operation was cancelled"

Hi, I’m trying to train a GAN on the mnist fashion dataset. But whenever I train the thing it keeps returning each epoch

W tensorflow/core/data/root_dataset.cc:163] Optimization loop failed: CANCELLED: Operation was cancelled W tensorflow/core/data/root_dataset.cc:163] Optimization loop failed: CANCELLED: Operation was cancelled W tensorflow/core/data/root_dataset.cc:163] Optimization loop failed: CANCELLED: Operation was cancelled W tensorflow/core/data/root_dataset.cc:163] Optimization loop failed: CANCELLED: Operation was cancelled W tensorflow/core/data/root_dataset.cc:163] Optimization loop failed: CANCELLED: Operation was cancelled W tensorflow/core/data/root_dataset.cc:163] Optimization loop failed: CANCELLED: Operation was cancelled 

It doesn’t seem to actually train every time l look at the generated images. Here is the code:

import tensorflow as tf from tensorflow import keras import matplotlib.pyplot as plt import numpy as np (X_train, y_train), (X_test, Y_test) = keras.datasets.fashion_mnist.load_data() X_train = X_train // 255.0 def plot_multiple_images(images, n_cols=None): n_cols = n_cols or len(images) n_rows = (len(images) - 1) // n_cols + 1 if images.shape[-1] == 1: images = np.squeeze(images, axis=-1) plt.figure(figsize=(n_cols, n_rows)) for index, image in enumerate(images): plt.subplot(n_rows, n_cols, index + 1) plt.imshow(image, cmap="binary") plt.axis("off") # np.random.seed(42) tf.random.set_seed(42) codings_size = 30 generator = keras.models.Sequential([ keras.layers.Dense(100, activation="selu", input_shape=[codings_size]), keras.layers.Dense(150, activation="selu"), keras.layers.Dense(28 * 28, activation="sigmoid"), keras.layers.Reshape([28, 28]) ]) discriminator = keras.models.Sequential([ keras.layers.Flatten(input_shape=[28, 28]), keras.layers.Dense(150, activation="selu"), keras.layers.Dense(100, activation="selu"), keras.layers.Dense(1, activation="sigmoid") ]) gan = keras.models.Sequential([generator, discriminator]) discriminator.compile(loss="binary_crossentropy", optimizer="rmsprop") discriminator.trainable = False gan.compile(loss="binary_crossentropy", optimizer="rmsprop") batch_size = 32 dataset = tf.data.Dataset.from_tensor_slices(X_train).shuffle(1000) dataset = dataset.batch(batch_size, drop_remainder=True).prefetch(1) batch_size = 32 def train_gan(gan, dataset, batch_size, codings_size, n_epochs=50): generator, discriminator = gan.layers for epoch in range(n_epochs): print("Epoch {}/{}".format(epoch + 1, n_epochs)) for X_batch in dataset: # phase 1 - training the discriminator noise = tf.random.normal(shape=[batch_size, codings_size]) generated_images = generator(noise) X_batch = tf.cast(X_batch, tf.float32) X_fake_and_real = tf.concat([generated_images, X_batch], axis=0) y1 = tf.constant([[0.]] * batch_size + [[1.]] * batch_size) discriminator.trainable = True discriminator.train_on_batch(X_fake_and_real, y1) # phase 2 - training the generator noise = tf.random.normal(shape=[batch_size, codings_size]) y2 = tf.constant([[1.]] * batch_size) discriminator.trainable = True gan.train_on_batch(noise, y2) # not shown plot_multiple_images(generated_images, 8) plt.show() train_gan(gan, dataset, batch_size, codings_size) 

I’m using Aurelien Geron’s “Hands-on Machine Learning with Scikit-Learn, Keras & Tensorflow”.

Here is the repository.

Hope this is the right sub for questions.