Updated 19/Nov/2021 by Yoshihisa Nitta 
In [ ]:
#! pip install tensorflow==2.7.0
In [ ]:
%tensorflow_version 2.x
import tensorflow as tf
print(tf.__version__)
AutoEncoder¶
-
Diederik P Kngma, Max Welling: Auto-Encoding Variational Bayes, 2013
https://arxiv.org/abs/1312.6114
「難しい事後分布を持つ連続潜在変数が存在し、さらにデータセットが大きい場合に、有効確率モデルを用いて効果的な推論と学習をどうやって行えばよいのだろうか」という問題がある。 この論文では、ある穏やかな(mild)微分可能条件下では適用できる、確率的変分推論と学習アルゴリズムを紹介する。 貢献する点は2点である。 「変分下限の再パラメータ化により、普通のSGDを用いてtraining可能な下限推定量が得られる」 「提案する下限推定器を用いて近似推論モデルを学習することによって、データポイント毎の連続潜在変数を持つ i.i.d データセットに対して事後推定が効率的に実行できる。」
In [ ]:
! nvidia-smi
! cat /proc/cpuinfo
! cat /etc/issue
! free -h
In [ ]:
from google.colab import drive
drive.mount('/content/drive')
In [ ]:
! ls /content/drive
Download the soure file from Google Drive or nw.tsuda.ac.jp¶
Basically, gdown from Google Drive. Download from nw.tsuda.ac.jp above only if the specifications of Google Drive change and you cannot download from Google Drive.
Google Drive または nw.tsuda.ac.jp からファイルをダウンロードする¶
基本的に Google Drive から gdown してください。 Google Drive の仕様が変わってダウンロードができない場合にのみ、nw.tsuda.ac.jp からダウンロードしてください。
In [ ]:
# Download source file
nw_path = './nw'
! rm -rf {nw_path}
! mkdir -p {nw_path}
if True: # from Google Drive
url_model = 'https://drive.google.com/uc?id=1ZDgWE7wmVwG_ZuQVUjuh_XHeIO-7Yn63'
! (cd {nw_path}; gdown {url_model})
else: # from nw.tsuda.ac.jp
URL_NW = 'https://nw.tsuda.ac.jp/lec/GoogleColab/pub'
url_model = f'{URL_NW}/models/AutoEncoder.py'
! wget -nd {url_model} -P {nw_path} # download to './nw/AutoEncoder.py'
In [ ]:
!cat {nw_path}/AutoEncoder.py
In [ ]:
import tensorflow as tf
import numpy as np
In [ ]:
# MNIST datasets
(x_train_raw, y_train_raw), (x_test_raw, y_test_raw) = tf.keras.datasets.mnist.load_data()
print(x_train_raw.shape)
print(y_train_raw.shape)
print(x_test_raw.shape)
print(y_test_raw.shape)
In [ ]:
x_train = x_train_raw.reshape(x_train_raw.shape+(1,)).astype('float32') / 255.0
x_test = x_test_raw.reshape(x_test_raw.shape+(1,)).astype('float32') / 255.0
print(x_train.shape)
print(x_test.shape)
In [ ]:
from nw.AutoEncoder import AutoEncoder
AE = AutoEncoder(
input_dim = (28, 28, 1),
encoder_conv_filters = [32, 64, 64, 64],
encoder_conv_kernel_size = [3, 3, 3, 3],
encoder_conv_strides = [1, 2, 2, 1],
decoder_conv_t_filters = [64, 64, 32, 1],
decoder_conv_t_kernel_size = [3, 3, 3, 3],
decoder_conv_t_strides = [1, 2, 2, 1],
z_dim = 2
)
In [ ]:
AE.encoder.summary()
In [ ]:
AE.decoder.summary()
In [ ]:
MAX_EPOCHS = 200
In [ ]:
learning_rate = 0.0005
In [ ]:
save_path1 = '/content/drive/MyDrive/ColabRun/AE01'
In [ ]:
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
AE.model.compile(optimizer=optimizer, loss=AutoEncoder.r_loss)
In [ ]:
# At first, train for a few epochs.
# まず、少ない回数 training してみる
history=AE.train_with_fit(
x_train,
x_train,
batch_size=32,
epochs = 3,
run_folder = save_path1,
validation_data = (x_test, x_test)
)
In [ ]:
print(history.history)
In [ ]:
# Load the trained states saved before
# 保存されている学習結果をロードする
AE_work = AutoEncoder.load(save_path1)
# display the epoch count of training
# training のepoch回数を表示する
print(AE_work.epoch)
In [ ]:
# Then, train for more epochs. The training continues from the current self.epoch to the epoches specified.
# 追加でtrainingする。保存されている現在のepoch数から始めて、指定したepochs までtrainingが進む。
AE_work.model.compile(optimizer, loss=AutoEncoder.r_loss)
history_work = AE_work.train_with_fit(
x_train,
x_train,
batch_size=32,
epochs=MAX_EPOCHS,
run_folder = save_path1,
validation_data=(x_test, x_test)
)
In [ ]:
# the return value contains the loss values in the additional training.
# 追加で行ったtraining時のlossが返り値に含まれる
print(len(history_work.history['loss']))
In [ ]:
loss1_1 = history.history['loss']
vloss1_1 = history.history['val_loss']
loss1_2 = history_work.history['loss']
vloss1_2 = history_work.history['val_loss']
loss1 = np.concatenate([loss1_1, loss1_2], axis=0)
val_loss1 = np.concatenate([vloss1_1, vloss1_2], axis=0)
In [ ]:
AutoEncoder.plot_history([loss1, val_loss1], ['loss', 'val_loss'])
In [ ]:
selected_indices = np.random.choice(range(len(x_test)), 10)
selected_images = x_test[selected_indices]
In [ ]:
z_points = AE_work.encoder.predict(selected_images)
reconst_images = AE_work.decoder.predict(z_points)
txts = [ f'{p[0]:.3f}, {p[1]:.3f}' for p in z_points ]
In [ ]:
%matplotlib inline
AutoEncoder.showImages(selected_images, reconst_images, txts, 1.4, 1.4)
(2) Training with tf.GradientTape() function.¶
Instead of using fit(), calculate the loss in your own train() function, find the gradients, and apply them to the variables.
The train_tf() function is speeding up by declaring <code>@tf.function</code> the compute_loss_and_grads() function.
(2) tf.GradientTape() 関数を使った学習¶
fit() 関数を使わずに、自分で記述した train() 関数内で loss を計算し、gradients を求めて、変数に適用する。
train_tf() 関数では、lossとgradientsの計算を行う compute_loss_and_grads() 関数を <code>@tf.function</code> 宣言することで高速化を図っている。
In [ ]:
save_path2 = '/content/drive/MyDrive/ColabRun/AE02/'
In [ ]:
from nw.AutoEncoder import AutoEncoder
AE2 = AutoEncoder(
input_dim = (28, 28, 1),
encoder_conv_filters = [32, 64, 64, 64],
encoder_conv_kernel_size = [3, 3, 3, 3],
encoder_conv_strides = [1, 2, 2, 1],
decoder_conv_t_filters = [64, 64, 32, 1],
decoder_conv_t_kernel_size = [3, 3, 3, 3],
decoder_conv_t_strides = [1, 2, 2, 1],
z_dim = 2
)
In [ ]:
optimizer2 = tf.keras.optimizers.Adam(learning_rate=learning_rate)
In [ ]:
# At first, train for a few epochs.
# まず、少ない回数 training してみる
loss2_1, vloss2_1 = AE2.train(
x_train,
x_train,
batch_size=32,
epochs = 3,
shuffle=True,
run_folder= save_path2,
optimizer = optimizer2,
save_epoch_interval=50,
validation_data=(x_test, x_test)
)
In [ ]:
# Load the parameters and the weights saved before.
# 保存したパラメータと、重みを読み込む。
AE2_work = AutoEncoder.load(save_path2)
print(AE2_work.epoch)
In [ ]:
# Additional Training.
# 追加でtrainingする。
# Compiles the part for loss and gradients fo train_tf() function into a graph of Tensorflow 2, so it is a little over twice as fast as train(). However, it is still nearly twice as slow as fit().
# train_tf() は loss と gradients を求める部分を tf のgraphにコンパイルしているので、train()よりも2倍強高速になっている。しかし、それでもfit()よりは2倍近く遅い。
loss2_2, vloss2_2 = AE2_work.train_tf(
x_train,
x_train,
batch_size=32,
epochs = MAX_EPOCHS,
shuffle=True,
run_folder= save_path2,
optimizer = optimizer2,
save_epoch_interval=50,
validation_data=(x_test, x_test)
)
In [ ]:
loss2 = np.concatenate([loss2_1, loss2_2], axis=0)
val_loss2 = np.concatenate([vloss2_1, vloss2_2], axis=0)
AutoEncoder.plot_history([loss2, val_loss2], ['loss', 'val_loss'])
In [ ]:
z_points2 = AE2_work.encoder.predict(selected_images)
reconst_images2 = AE2_work.decoder.predict(z_points2)
txts2 = [ f'{p[0]:.3f}, {p[1]:.3f}' for p in z_points2 ]
In [ ]:
%matplotlib inline
AutoEncoder.showImages(selected_images, reconst_images2, txts2, 1.4, 1.4)
(3) Trainig with tf.GradientTape() function and Learning rate decay¶
Calculate the loss and gradients with the tf.GradientTape() function, and apply the gradients to the variables.
In addition, perform Learning rate decay in the optimizer.
[Caution] Note that if you call the save_image() function in the training, encoder.predict() and decoder.predict() will work and the execution will be slow.
(3) tf.GradientTape() 関数と学習率減衰を使った学習¶
tf.GradientTape() 関数を使って loss と gradients を計算して、gradients を変数に適用する。
さらに、optimizer において Learning rate decay を行う。
(注意) trainingの途中で save_images()関数を呼び出すと、 encoder.predict()と decoder.predict() が動作して、実行が非常に遅くなるので注意すること。
In [ ]:
save_path3 = '/content/drive/MyDrive/ColabRun/AE03/'
In [ ]:
from nw.AutoEncoder import AutoEncoder
AE3 = AutoEncoder(
input_dim = (28, 28, 1),
encoder_conv_filters = [32, 64, 64, 64],
encoder_conv_kernel_size = [3, 3, 3, 3],
encoder_conv_strides = [1, 2, 2, 1],
decoder_conv_t_filters = [64, 64, 32, 1],
decoder_conv_t_kernel_size = [3, 3, 3, 3],
decoder_conv_t_strides = [1, 2, 2, 1],
z_dim = 2
)
In [ ]:
# initial_learning_rate * decay_rate ^ (step // decay_steps)
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate = learning_rate,
decay_steps = 1000,
decay_rate=0.96
)
optimizer3 = tf.keras.optimizers.Adam(learning_rate=lr_schedule)
In [ ]:
# At first, train for a few epochs.
# まず、少ない回数 training してみる
loss3_1, vloss3_1 = AE3.train(
x_train,
x_train,
batch_size=32,
epochs = 3,
shuffle=True,
run_folder=save_path3,
optimizer = optimizer3,
save_epoch_interval=50,
validation_data=(x_test, x_test)
)
In [ ]:
# Load the parameters and the weights saved before.
# 保存したパラメータと、重みを読み込む。
AE3_work = AutoEncoder.load(save_path3)
print(AE3_work.epoch)
In [ ]:
# Additional Training.
# 追加でtrainingする。
# Compiles the part for loss and gradients fo train_tf() function into a graph of Tensorflow 2, so it is a little over twice as fast as train(). However, it is still nearly twice as slow as fit().
# train_tf() は loss と gradients を求める部分を tf のgraphにコンパイルしているので、train()よりも2倍強高速になっている。しかし、それでもfit()よりは2倍近く遅い。
loss3_2, vloss3_2 = AE3_work.train_tf(
x_train,
x_train,
batch_size=32,
epochs = MAX_EPOCHS,
shuffle=True,
run_folder= save_path3,
optimizer = optimizer3,
save_epoch_interval=50,
validation_data=(x_test, x_test)
)
In [ ]:
loss3 = np.concatenate([loss3_1, loss3_2], axis=0)
val_loss3 = np.concatenate([vloss3_1, vloss3_2], axis=0)
AutoEncoder.plot_history([loss3, val_loss3], ['loss', 'val_loss'])
In [ ]:
z_points3 = AE3_work.encoder.predict(selected_images)
reconst_images3 = AE3_work.decoder.predict(z_points3)
txts3 = [ f'{p[0]:.3f}, {p[1]:.3f}' for p in z_points3 ]
In [ ]:
%matplotlib inline
AutoEncoder.showImages(selected_images, reconst_images3, txts3, 1.4, 1.4)
In [ ]: