PyTorch basics

Outline

Source: https://pytorch-geometric.readthedocs.io/en/latest/get_started/colabs.html

  • Datasets
  • Models
  • Losses
  • Optimizers

Official resources:

Here no details, mostly visualization, with linear regression.

Imports

import numpy as np
import torch
import matplotlib.pyplot as plt
from matplotlib import colors
plt.rcParams.update({'font.size': 16})
x = torch.rand(20, 5)
x
((x.norm(dim=1) - 1).abs() < 1e-10).float()
tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])

Datasets

A raw dataset

The data are obtained from the model

\[ x \mapsto y:=Ax + b \]

with \(x\in\mathbb R^{input\_dim}\) and \(y\in\mathbb R^{output\_dim}\).

input_dim = 1
output_dim = 1

A = 2 * np.random.rand(output_dim, input_dim) - 1
b = 2 * np.random.rand(output_dim) - 1

true_model = lambda x: A @ x + b

We get a training set by randomly sampling and noisy observations

\[ \begin{align*} &x_i \sim U([-1, 1])\\ &y_i = A x_i + b + \nu_i \end{align*} \]

n_train = 1000
noise_level = 0.04

# Generate a random set of n_train samples
X_train = np.random.rand(n_train, input_dim)
y_train = np.array([true_model(x) for x in X_train])

# Add some noise
y_train += noise_level * np.random.standard_normal(size=y_train.shape)
if input_dim == output_dim == 1:
    fig = plt.figure()
    fig.clf()
    ax = fig.gca()
    ax.plot(X_train, y_train, '.')
    ax.grid(True)
    ax.set_xlabel('X_train')
    ax.set_ylabel('y_train')

PyTorch Dataset

#%% Dataset to manage vector to vector data
class VectorialDataset(torch.utils.data.Dataset):
    def __init__(self, input_data, output_data):
        super(VectorialDataset, self).__init__()
        self.input_data = torch.tensor(input_data.astype('f'))
        self.output_data = torch.tensor(output_data.astype('f'))

    def __len__(self):
        return self.input_data.shape[0]

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        sample = (self.input_data[idx, :],
                  self.output_data[idx, :])
        return sample
training_set = VectorialDataset(input_data=X_train, output_data=y_train)
len(training_set)
1000
training_set[10:12]
(tensor([[0.9493],
         [0.6869]]),
 tensor([[-1.5009],
         [-1.3625]]))

Here almost useless, but think about e.g. images.

PyTorch DataLoader

batch_size = 120
train_loader = torch.utils.data.DataLoader(training_set,
                                           batch_size=batch_size,
                                           shuffle=True)

The role of batch_size:

for idx, batch in enumerate(train_loader):
    print('Batch n. %2d: input size=%s, output size=%s' % (idx+1, batch[0].shape, batch[1].shape))
Batch n.  1: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  2: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  3: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  4: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  5: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  6: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  7: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  8: input size=torch.Size([120, 1]), output size=torch.Size([120, 1])
Batch n.  9: input size=torch.Size([40, 1]), output size=torch.Size([40, 1])
8 * 120
960

The role of shuffle:

first_batch = []

for epoch in range(2):
    for idx, batch in enumerate(train_loader):
        if idx == 0:
            first_batch.append(batch)

np.c_[X_train[:batch_size], first_batch[0][0].numpy(), first_batch[1][0].numpy()]
array([[0.60644203, 0.72176361, 0.20528394],
       [0.20063265, 0.87918973, 0.94030756],
       [0.6147621 , 0.87515396, 0.07383709],
       [0.01099506, 0.62517351, 0.12932838],
       [0.06699535, 0.03348689, 0.22834778],
       [0.28576401, 0.9150539 , 0.95221508],
       [0.22133355, 0.91566157, 0.88769048],
       [0.90814814, 0.36945143, 0.46483845],
       [0.6954035 , 0.53236091, 0.86544257],
       [0.20337042, 0.44153246, 0.4435882 ],
       [0.94931462, 0.94847393, 0.46798134],
       [0.68689474, 0.46421856, 0.77144283],
       [0.86262639, 0.86281133, 0.743186  ],
       [0.66824892, 0.38669878, 0.50947791],
       [0.07383709, 0.17755535, 0.77268142],
       [0.5311849 , 0.56230175, 0.89632595],
       [0.83337218, 0.34319821, 0.57058495],
       [0.75386386, 0.67870641, 0.65680295],
       [0.73545148, 0.20337042, 0.83965939],
       [0.88236311, 0.43531543, 0.6707992 ],
       [0.39882423, 0.1109324 , 0.71643633],
       [0.58938745, 0.74548542, 0.04050675],
       [0.07481249, 0.83810669, 0.21418592],
       [0.40622471, 0.81048596, 0.39649394],
       [0.92722397, 0.91158998, 0.04317835],
       [0.8584473 , 0.94127363, 0.39913154],
       [0.26920117, 0.90823168, 0.59952587],
       [0.05387456, 0.42325148, 0.7901687 ],
       [0.83869014, 0.27029511, 0.80333126],
       [0.9905177 , 0.63552886, 0.41777459],
       [0.16855289, 0.46621531, 0.43919873],
       [0.26799051, 0.4242816 , 0.0711002 ],
       [0.03176938, 0.38176569, 0.98920375],
       [0.30626154, 0.07481249, 0.92059278],
       [0.52782559, 0.30331537, 0.62306315],
       [0.39692093, 0.47368419, 0.27568358],
       [0.81569599, 0.02610275, 0.69333148],
       [0.65287379, 0.3578262 , 0.94272584],
       [0.01980153, 0.25089583, 0.06133046],
       [0.98920373, 0.06371544, 0.44706064],
       [0.8267019 , 0.02967571, 0.73912191],
       [0.41706685, 0.16748054, 0.10925003],
       [0.21002921, 0.25484258, 0.28775713],
       [0.90293698, 0.43267664, 0.56066865],
       [0.77082519, 0.41439366, 0.53760457],
       [0.91253632, 0.39927718, 0.60219854],
       [0.57586975, 0.59146208, 0.87213171],
       [0.88769048, 0.36215627, 0.69944972],
       [0.26177647, 0.82743722, 0.52128166],
       [0.81839254, 0.815696  , 0.7972796 ],
       [0.8914107 , 0.2658999 , 0.92999226],
       [0.7268874 , 0.36035451, 0.71367186],
       [0.47174742, 0.2840862 , 0.87851882],
       [0.79610382, 0.04591979, 0.38779262],
       [0.8220842 , 0.85620123, 0.90232348],
       [0.71643634, 0.52469993, 0.29921648],
       [0.74318597, 0.47862551, 0.96101135],
       [0.86577598, 0.16242123, 0.13386725],
       [0.76178862, 0.94576132, 0.46689343],
       [0.40742536, 0.60341704, 0.21002921],
       [0.68308181, 0.38553789, 0.71847188],
       [0.30598878, 0.65680832, 0.40524086],
       [0.61663578, 0.26040652, 0.24785757],
       [0.95074046, 0.83309424, 0.9464947 ],
       [0.24202712, 0.16343524, 0.93753624],
       [0.34630605, 0.11649692, 0.89906967],
       [0.29616019, 0.82340431, 0.79744017],
       [0.48492434, 0.96443754, 0.68164057],
       [0.30679355, 0.62452585, 0.08947403],
       [0.81026962, 0.98967212, 0.99049819],
       [0.87918974, 0.89141071, 0.96239102],
       [0.68376444, 0.70030856, 0.04591979],
       [0.01728177, 0.12982078, 0.03222714],
       [0.5847346 , 0.20773079, 0.66903222],
       [0.3855379 , 0.76285899, 0.70062685],
       [0.69324651, 0.48147103, 0.53027409],
       [0.72638407, 0.71643633, 0.81839252],
       [0.04340754, 0.13605507, 0.89705622],
       [0.85693218, 0.36119753, 0.01689854],
       [0.90317674, 0.28372869, 0.08969983],
       [0.8381067 , 0.36108783, 0.67363423],
       [0.92816174, 0.12811057, 0.6497916 ],
       [0.27887548, 0.02964934, 0.82208419],
       [0.43517069, 0.60331875, 0.17433779],
       [0.76693588, 0.41211167, 0.25985774],
       [0.58239489, 0.89780653, 0.46792969],
       [0.92510619, 0.0711002 , 0.85101849],
       [0.23741723, 0.41568658, 0.79685956],
       [0.82743724, 0.24690489, 0.56617481],
       [0.0420613 , 0.28756037, 0.36119753],
       [0.65165534, 0.58968705, 0.88549757],
       [0.68087389, 0.40740794, 0.26166803],
       [0.53236092, 0.90814811, 0.96314073],
       [0.46007617, 0.92167944, 0.73320228],
       [0.53027408, 0.68141228, 0.59730726],
       [0.55093432, 0.9453178 , 0.01941664],
       [0.94253602, 0.00801179, 0.76033908],
       [0.8721317 , 0.33238789, 0.01763334],
       [0.32037741, 0.15823166, 0.0096136 ],
       [0.08549398, 0.4691546 , 0.05822677],
       [0.82340431, 0.5624494 , 0.81795985],
       [0.67062066, 0.17329524, 0.81026959],
       [0.87603401, 0.68304455, 0.95495564],
       [0.8629597 , 0.62270319, 0.58968705],
       [0.41067377, 0.10788412, 0.6037848 ],
       [0.5423717 , 0.40505263, 0.83236474],
       [0.06448759, 0.56171715, 0.67423075],
       [0.04137664, 0.47860199, 0.9527849 ],
       [0.74052561, 0.23295914, 0.47291851],
       [0.12720158, 0.75596493, 0.09361077],
       [0.56171713, 0.85142255, 0.70030856],
       [0.41473388, 0.04317835, 0.29211819],
       [0.12932839, 0.7625168 , 0.58542591],
       [0.2069174 , 0.18442018, 0.54237169],
       [0.96173181, 0.51359886, 0.99478054],
       [0.94568202, 0.36403251, 0.62867373],
       [0.51482581, 0.21834005, 0.88440764],
       [0.20494642, 0.06899509, 0.70275831],
       [0.58292683, 0.41325051, 0.97383028],
       [0.83058274, 0.50676352, 0.70857292]])

Models

We implement a linear model \[x \mapsto model(x):=A x + b\]

with \(A\in \mathbb{R}^{input\_dim\times output\_dim}\), \(b\in\mathbb{R}^{output\_dim}\).

import torch.nn as nn
import torch

#%% Linear layer
class LinearModel(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(LinearModel, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim

        self.linear = nn.Linear(self.input_dim, self.output_dim, bias=True)

    def forward(self, x):
        out = self.linear(x)
        return out

    def reset(self):
        self.linear.reset_parameters()
model = LinearModel(input_dim, output_dim)

Model inspection

print(model)
LinearModel(
  (linear): Linear(in_features=1, out_features=1, bias=True)
)

See pytorch.org/docs/stable/nn.html for many other layers.

list(model.parameters())
[Parameter containing:
 tensor([[-0.1260]], requires_grad=True),
 Parameter containing:
 tensor([-0.8922], requires_grad=True)]
model.linear.weight
Parameter containing:
tensor([[-0.1260]], requires_grad=True)
model.linear.bias
Parameter containing:
tensor([-0.8922], requires_grad=True)

The forward method

x = torch.randn(5, input_dim)
model.forward(x)
tensor([[-0.8423],
        [-0.9081],
        [-1.0015],
        [-0.9104],
        [-0.7193]], grad_fn=<AddmmBackward0>)
[model.linear.weight @ xx + model.linear.bias for xx in x]
[tensor([-0.8423], grad_fn=<AddBackward0>),
 tensor([-0.9081], grad_fn=<AddBackward0>),
 tensor([-1.0015], grad_fn=<AddBackward0>),
 tensor([-0.9104], grad_fn=<AddBackward0>),
 tensor([-0.7193], grad_fn=<AddBackward0>)]
if input_dim == output_dim == 1:
    fig = plt.figure()
    fig.clf()
    ax = fig.gca()
    ax.plot(training_set.input_data, training_set.output_data, '.')
    ax.plot(training_set.input_data, model.forward(training_set.input_data).detach().numpy(), '.')
    ax.grid(True)
    ax.set_xlabel('X_train')
    ax.legend(['y_train', 'model(X_train)'])

Losses

The MSE loss

The MSE loss is \[ L(y, y') = \|y-y'\|_2^2 \]

import torch.nn as nn
loss_fun = nn.MSELoss(reduction='mean')

More at pytorch.org/docs/stable/nn.html#loss-functions

x = torch.tensor(np.array([1, 2, 1]).astype('f'))
z = torch.tensor(np.array([0, 0, 0]).astype('f'))
loss_fun(x, z)
tensor(2.)

Giving a score to the model (parameters) given the training set

The loss for one data pair \((x_i, y_i)\) is \[ L(model(x_i), y_i) \]

The cumulative (average) loss on the training set is \[ L(X_{train}, y_{train}):=\frac{1}{n_{train}} \sum_{i=1}^{n_{train}} L(model(x_i), y_i) \]

if input_dim == output_dim == 1:

    state_dict = model.state_dict()

    ww, bb = np.meshgrid(np.linspace(-2, 2, 30), np.linspace(-2, 2, 30))

    loss_values = 0 * ww
    for i in range(ww.shape[0]):
        for j in range(ww.shape[1]):
            state_dict['linear.weight'] = torch.tensor([[ww[i, j]]])
            state_dict['linear.bias'] = torch.tensor([bb[i, j]])
            model.load_state_dict(state_dict)
            loss_values[i, j] = loss_fun(model.forward(training_set.input_data),  training_set.output_data)

    fig = plt.figure(figsize=(10, 8))
    fig.clf()
    ax = fig.gca()
    levels = np.logspace(np.log(np.min(loss_values)), np.log(np.max(loss_values)), 20)
    c=ax.contourf(ww, bb, loss_values, levels=levels, norm=colors.LogNorm())
    plt.colorbar(c)
    ax.plot(A[0], b, 'r*', markersize=10)
    ax.set_ylabel('bias')
    ax.set_xlabel('weight')
    ax.legend(['(A, b)'])

    ax.grid(True)

Optimization

Goal: minimize the loss given the data: \[ \min\limits_{par \in model} L(X_{train}, y_{train}) =\min\limits_{par \in model} \frac{1}{n_{train}} \sum_{i=1}^{n_{train}} L(x_i, model(x_i)) \]

In this case \(par =\{A, b\}\).

Iterative gradient based optimization: \[\begin{align*} par^{(0)} &= par_0\\ par^{(k+1)} &= par^{(k)} - \eta^{(k)} \nabla_{par} L(X_{train}, y_{train}) \end{align*}\]
with learning rate \(\eta^{(k)}\).

Gradients

The model is \[ model(x) = A x + b \]

With \(input\_dim = output\_dim = 1\) we have \(A:=a\in\mathbb{R},\ b\in\mathbb{R}\).

For a data pair \((x, y)\) the loss is

\[ \begin{align*} L(x, y) &= \|model(x)-y\|_2^2\\ &= (model(x)-y)^2\\ &= (a x + b - y)^2\\ &= a^2 x^2 + 2 a b x - 2 a xy + b^2 -2 b y + y^2. \end{align*} \]

We can compute

\[ \begin{align*} \nabla_a L(x, y) &= 2 a x^2 + 2 b x - 2 xy =2 x (a x + b - y)\\ \nabla_b L(x, y) &= 2 a x + 2 b - 2 y = 2 (ax + b - y). \end{align*} \]

x = torch.randn(1, input_dim)
y =  torch.randn(1, output_dim)

model.zero_grad()
loss = loss_fun(model.forward(x),  y)
loss.backward()
if input_dim == output_dim == 1:
    print(model.linear.weight.grad)
    print(2 * x * (model.linear.weight * x + model.linear.bias - y))

    print(model.linear.bias.grad)
    print(2 * (model.linear.weight * x + model.linear.bias - y))
tensor([[-0.6195]])
tensor([[-0.6195]], grad_fn=<MulBackward0>)
tensor([3.3432])
tensor([[3.3432]], grad_fn=<MulBackward0>)

Handmade optimization

if input_dim == output_dim == 1:

    num_iter = 200
    lr = 0.5 # 0.01

    train_hist = {}
    train_hist['weight'] = []
    train_hist['bias'] = []

    model.reset()
    state_dict = model.state_dict()

    for _ in range(num_iter):

        model.zero_grad()
        loss = loss_fun(model.forward(training_set.input_data), training_set.output_data)
        loss.backward()

        w = model.linear.weight.item()
        b = model.linear.bias.item()

        dw = model.linear.weight.grad.item()
        db = model.linear.bias.grad.item()

        state_dict['linear.weight'] += torch.tensor([-lr * dw])
        state_dict['linear.bias'] += torch.tensor([-lr * db])
        model.load_state_dict(state_dict)

        train_hist['weight'].append(w)
        train_hist['bias'].append(b)

    for label in train_hist:
        train_hist[label] = np.array(train_hist[label])
if input_dim == output_dim == 1:
    fig = plt.figure(figsize=(8, 8))
    fig.clf()
    ax = fig.gca()
    levels = np.logspace(np.log(np.min(loss_values)), np.log(np.max(loss_values)), 20)
    ax.contourf(ww, bb, loss_values, levels=levels, norm=colors.LogNorm())
    ax.set_xlabel('weight')
    ax.set_ylabel('bias')
    ax.grid(True)
    ax.set_xlim(-2, 2)
    ax.set_ylim(-2, 2)

    ax.plot(train_hist['weight'], train_hist['bias'], '.-b')
    ax.plot(A[0], b, 'r*', markersize=10)

    #ax.legend(['optim', '(A, b)'])

Stochastic GD

  • Avoid loading the full training set
  • Avoid evaluating the model on the full training set

At every step, compute the loss on a batch of data \((X^{(k)}, y^{(k)})\sim (X_{train}, y_{train})\):

\[ \begin{align*} L(X^{(k)}, y^{(k)}) := \sum_{(x, y) \in (X^{(k)}, y^{(k)})} L(x, model(x)). \end{align*} \]

and then update with this approximated gradient:

\[ \begin{align*} par^{(k+1)} &= par^{(k)} - \eta^{(k)} \nabla_{par} L(X^{(k)}, y^{(k)}). \end{align*} \]

PyTorch optim

lr = 0.1
weight_decay = 5e-4
optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)

More at pytorch.org/docs/stable/optim.html.

n_epochs = 100
train_hist = {}
train_hist['loss'] = []

if input_dim == output_dim == 1:
    train_hist['weight'] = []
    train_hist['bias'] = []

# Initialize training
model.reset()
model.train()

for epoch in range(n_epochs):
    for idx, batch in enumerate(train_loader):
        optimizer.zero_grad()
        loss = loss_fun(model.forward(batch[0]),  batch[1])
        loss.backward()
        optimizer.step()

        train_hist['loss'].append(loss.item())
        if input_dim == output_dim == 1:
            train_hist['weight'].append(model.linear.weight.item())
            train_hist['bias'].append(model.linear.bias.item())

        print('[Epoch %4d/%4d] [Batch %4d/%4d] Loss: % 2.2e' % (epoch + 1, n_epochs,
                                                                idx + 1, len(train_loader),
                                                                loss.item()))

model.eval()
[Epoch    1/ 100] [Batch    1/   9] Loss:  3.12e+00
[Epoch    1/ 100] [Batch    2/   9] Loss:  2.63e+00
[Epoch    1/ 100] [Batch    3/   9] Loss:  2.20e+00
[Epoch    1/ 100] [Batch    4/   9] Loss:  1.71e+00
[Epoch    1/ 100] [Batch    5/   9] Loss:  1.35e+00
[Epoch    1/ 100] [Batch    6/   9] Loss:  1.06e+00
[Epoch    1/ 100] [Batch    7/   9] Loss:  7.33e-01
[Epoch    1/ 100] [Batch    8/   9] Loss:  5.48e-01
[Epoch    1/ 100] [Batch    9/   9] Loss:  3.53e-01
[Epoch    2/ 100] [Batch    1/   9] Loss:  2.09e-01
[Epoch    2/ 100] [Batch    2/   9] Loss:  1.13e-01
[Epoch    2/ 100] [Batch    3/   9] Loss:  4.48e-02
[Epoch    2/ 100] [Batch    4/   9] Loss:  1.34e-02
[Epoch    2/ 100] [Batch    5/   9] Loss:  4.57e-03
[Epoch    2/ 100] [Batch    6/   9] Loss:  1.81e-02
[Epoch    2/ 100] [Batch    7/   9] Loss:  3.92e-02
[Epoch    2/ 100] [Batch    8/   9] Loss:  7.31e-02
[Epoch    2/ 100] [Batch    9/   9] Loss:  1.00e-01
[Epoch    3/ 100] [Batch    1/   9] Loss:  1.34e-01
[Epoch    3/ 100] [Batch    2/   9] Loss:  1.57e-01
[Epoch    3/ 100] [Batch    3/   9] Loss:  1.97e-01
[Epoch    3/ 100] [Batch    4/   9] Loss:  1.81e-01
[Epoch    3/ 100] [Batch    5/   9] Loss:  1.96e-01
[Epoch    3/ 100] [Batch    6/   9] Loss:  1.93e-01
[Epoch    3/ 100] [Batch    7/   9] Loss:  1.78e-01
[Epoch    3/ 100] [Batch    8/   9] Loss:  1.53e-01
[Epoch    3/ 100] [Batch    9/   9] Loss:  1.04e-01
[Epoch    4/ 100] [Batch    1/   9] Loss:  1.13e-01
[Epoch    4/ 100] [Batch    2/   9] Loss:  8.12e-02
[Epoch    4/ 100] [Batch    3/   9] Loss:  6.65e-02
[Epoch    4/ 100] [Batch    4/   9] Loss:  4.33e-02
[Epoch    4/ 100] [Batch    5/   9] Loss:  3.13e-02
[Epoch    4/ 100] [Batch    6/   9] Loss:  1.74e-02
[Epoch    4/ 100] [Batch    7/   9] Loss:  1.04e-02
[Epoch    4/ 100] [Batch    8/   9] Loss:  5.42e-03
[Epoch    4/ 100] [Batch    9/   9] Loss:  4.67e-03
[Epoch    5/ 100] [Batch    1/   9] Loss:  5.21e-03
[Epoch    5/ 100] [Batch    2/   9] Loss:  9.43e-03
[Epoch    5/ 100] [Batch    3/   9] Loss:  1.09e-02
[Epoch    5/ 100] [Batch    4/   9] Loss:  1.48e-02
[Epoch    5/ 100] [Batch    5/   9] Loss:  1.76e-02
[Epoch    5/ 100] [Batch    6/   9] Loss:  2.10e-02
[Epoch    5/ 100] [Batch    7/   9] Loss:  2.34e-02
[Epoch    5/ 100] [Batch    8/   9] Loss:  2.36e-02
[Epoch    5/ 100] [Batch    9/   9] Loss:  2.43e-02
[Epoch    6/ 100] [Batch    1/   9] Loss:  2.24e-02
[Epoch    6/ 100] [Batch    2/   9] Loss:  2.25e-02
[Epoch    6/ 100] [Batch    3/   9] Loss:  1.81e-02
[Epoch    6/ 100] [Batch    4/   9] Loss:  1.49e-02
[Epoch    6/ 100] [Batch    5/   9] Loss:  1.01e-02
[Epoch    6/ 100] [Batch    6/   9] Loss:  9.36e-03
[Epoch    6/ 100] [Batch    7/   9] Loss:  6.83e-03
[Epoch    6/ 100] [Batch    8/   9] Loss:  4.11e-03
[Epoch    6/ 100] [Batch    9/   9] Loss:  3.10e-03
[Epoch    7/ 100] [Batch    1/   9] Loss:  2.16e-03
[Epoch    7/ 100] [Batch    2/   9] Loss:  2.32e-03
[Epoch    7/ 100] [Batch    3/   9] Loss:  3.20e-03
[Epoch    7/ 100] [Batch    4/   9] Loss:  2.91e-03
[Epoch    7/ 100] [Batch    5/   9] Loss:  3.62e-03
[Epoch    7/ 100] [Batch    6/   9] Loss:  4.41e-03
[Epoch    7/ 100] [Batch    7/   9] Loss:  5.01e-03
[Epoch    7/ 100] [Batch    8/   9] Loss:  3.91e-03
[Epoch    7/ 100] [Batch    9/   9] Loss:  5.78e-03
[Epoch    8/ 100] [Batch    1/   9] Loss:  6.04e-03
[Epoch    8/ 100] [Batch    2/   9] Loss:  4.30e-03
[Epoch    8/ 100] [Batch    3/   9] Loss:  3.99e-03
[Epoch    8/ 100] [Batch    4/   9] Loss:  3.27e-03
[Epoch    8/ 100] [Batch    5/   9] Loss:  4.35e-03
[Epoch    8/ 100] [Batch    6/   9] Loss:  3.39e-03
[Epoch    8/ 100] [Batch    7/   9] Loss:  2.66e-03
[Epoch    8/ 100] [Batch    8/   9] Loss:  2.41e-03
[Epoch    8/ 100] [Batch    9/   9] Loss:  2.04e-03
[Epoch    9/ 100] [Batch    1/   9] Loss:  1.57e-03
[Epoch    9/ 100] [Batch    2/   9] Loss:  1.90e-03
[Epoch    9/ 100] [Batch    3/   9] Loss:  1.74e-03
[Epoch    9/ 100] [Batch    4/   9] Loss:  2.22e-03
[Epoch    9/ 100] [Batch    5/   9] Loss:  2.31e-03
[Epoch    9/ 100] [Batch    6/   9] Loss:  2.61e-03
[Epoch    9/ 100] [Batch    7/   9] Loss:  1.95e-03
[Epoch    9/ 100] [Batch    8/   9] Loss:  2.50e-03
[Epoch    9/ 100] [Batch    9/   9] Loss:  2.21e-03
[Epoch   10/ 100] [Batch    1/   9] Loss:  2.85e-03
[Epoch   10/ 100] [Batch    2/   9] Loss:  1.98e-03
[Epoch   10/ 100] [Batch    3/   9] Loss:  1.92e-03
[Epoch   10/ 100] [Batch    4/   9] Loss:  2.16e-03
[Epoch   10/ 100] [Batch    5/   9] Loss:  1.99e-03
[Epoch   10/ 100] [Batch    6/   9] Loss:  1.51e-03
[Epoch   10/ 100] [Batch    7/   9] Loss:  1.70e-03
[Epoch   10/ 100] [Batch    8/   9] Loss:  2.19e-03
[Epoch   10/ 100] [Batch    9/   9] Loss:  9.30e-04
[Epoch   11/ 100] [Batch    1/   9] Loss:  1.64e-03
[Epoch   11/ 100] [Batch    2/   9] Loss:  2.06e-03
[Epoch   11/ 100] [Batch    3/   9] Loss:  1.60e-03
[Epoch   11/ 100] [Batch    4/   9] Loss:  1.77e-03
[Epoch   11/ 100] [Batch    5/   9] Loss:  1.67e-03
[Epoch   11/ 100] [Batch    6/   9] Loss:  1.48e-03
[Epoch   11/ 100] [Batch    7/   9] Loss:  1.90e-03
[Epoch   11/ 100] [Batch    8/   9] Loss:  1.89e-03
[Epoch   11/ 100] [Batch    9/   9] Loss:  1.08e-03
[Epoch   12/ 100] [Batch    1/   9] Loss:  1.38e-03
[Epoch   12/ 100] [Batch    2/   9] Loss:  1.57e-03
[Epoch   12/ 100] [Batch    3/   9] Loss:  1.50e-03
[Epoch   12/ 100] [Batch    4/   9] Loss:  1.87e-03
[Epoch   12/ 100] [Batch    5/   9] Loss:  1.83e-03
[Epoch   12/ 100] [Batch    6/   9] Loss:  1.60e-03
[Epoch   12/ 100] [Batch    7/   9] Loss:  1.72e-03
[Epoch   12/ 100] [Batch    8/   9] Loss:  1.68e-03
[Epoch   12/ 100] [Batch    9/   9] Loss:  1.40e-03
[Epoch   13/ 100] [Batch    1/   9] Loss:  1.93e-03
[Epoch   13/ 100] [Batch    2/   9] Loss:  1.55e-03
[Epoch   13/ 100] [Batch    3/   9] Loss:  1.48e-03
[Epoch   13/ 100] [Batch    4/   9] Loss:  1.90e-03
[Epoch   13/ 100] [Batch    5/   9] Loss:  1.56e-03
[Epoch   13/ 100] [Batch    6/   9] Loss:  1.65e-03
[Epoch   13/ 100] [Batch    7/   9] Loss:  1.59e-03
[Epoch   13/ 100] [Batch    8/   9] Loss:  1.33e-03
[Epoch   13/ 100] [Batch    9/   9] Loss:  1.15e-03
[Epoch   14/ 100] [Batch    1/   9] Loss:  1.43e-03
[Epoch   14/ 100] [Batch    2/   9] Loss:  1.55e-03
[Epoch   14/ 100] [Batch    3/   9] Loss:  1.39e-03
[Epoch   14/ 100] [Batch    4/   9] Loss:  1.65e-03
[Epoch   14/ 100] [Batch    5/   9] Loss:  1.63e-03
[Epoch   14/ 100] [Batch    6/   9] Loss:  1.63e-03
[Epoch   14/ 100] [Batch    7/   9] Loss:  1.47e-03
[Epoch   14/ 100] [Batch    8/   9] Loss:  1.91e-03
[Epoch   14/ 100] [Batch    9/   9] Loss:  1.70e-03
[Epoch   15/ 100] [Batch    1/   9] Loss:  1.42e-03
[Epoch   15/ 100] [Batch    2/   9] Loss:  1.44e-03
[Epoch   15/ 100] [Batch    3/   9] Loss:  1.68e-03
[Epoch   15/ 100] [Batch    4/   9] Loss:  1.67e-03
[Epoch   15/ 100] [Batch    5/   9] Loss:  1.68e-03
[Epoch   15/ 100] [Batch    6/   9] Loss:  1.65e-03
[Epoch   15/ 100] [Batch    7/   9] Loss:  1.66e-03
[Epoch   15/ 100] [Batch    8/   9] Loss:  1.28e-03
[Epoch   15/ 100] [Batch    9/   9] Loss:  1.90e-03
[Epoch   16/ 100] [Batch    1/   9] Loss:  1.22e-03
[Epoch   16/ 100] [Batch    2/   9] Loss:  1.67e-03
[Epoch   16/ 100] [Batch    3/   9] Loss:  1.59e-03
[Epoch   16/ 100] [Batch    4/   9] Loss:  1.61e-03
[Epoch   16/ 100] [Batch    5/   9] Loss:  1.44e-03
[Epoch   16/ 100] [Batch    6/   9] Loss:  1.75e-03
[Epoch   16/ 100] [Batch    7/   9] Loss:  1.84e-03
[Epoch   16/ 100] [Batch    8/   9] Loss:  1.46e-03
[Epoch   16/ 100] [Batch    9/   9] Loss:  1.48e-03
[Epoch   17/ 100] [Batch    1/   9] Loss:  1.27e-03
[Epoch   17/ 100] [Batch    2/   9] Loss:  1.55e-03
[Epoch   17/ 100] [Batch    3/   9] Loss:  1.82e-03
[Epoch   17/ 100] [Batch    4/   9] Loss:  1.59e-03
[Epoch   17/ 100] [Batch    5/   9] Loss:  1.74e-03
[Epoch   17/ 100] [Batch    6/   9] Loss:  1.11e-03
[Epoch   17/ 100] [Batch    7/   9] Loss:  1.89e-03
[Epoch   17/ 100] [Batch    8/   9] Loss:  1.64e-03
[Epoch   17/ 100] [Batch    9/   9] Loss:  1.32e-03
[Epoch   18/ 100] [Batch    1/   9] Loss:  1.66e-03
[Epoch   18/ 100] [Batch    2/   9] Loss:  1.58e-03
[Epoch   18/ 100] [Batch    3/   9] Loss:  1.50e-03
[Epoch   18/ 100] [Batch    4/   9] Loss:  1.71e-03
[Epoch   18/ 100] [Batch    5/   9] Loss:  1.44e-03
[Epoch   18/ 100] [Batch    6/   9] Loss:  1.37e-03
[Epoch   18/ 100] [Batch    7/   9] Loss:  1.44e-03
[Epoch   18/ 100] [Batch    8/   9] Loss:  1.73e-03
[Epoch   18/ 100] [Batch    9/   9] Loss:  1.84e-03
[Epoch   19/ 100] [Batch    1/   9] Loss:  1.78e-03
[Epoch   19/ 100] [Batch    2/   9] Loss:  1.42e-03
[Epoch   19/ 100] [Batch    3/   9] Loss:  1.89e-03
[Epoch   19/ 100] [Batch    4/   9] Loss:  1.34e-03
[Epoch   19/ 100] [Batch    5/   9] Loss:  1.58e-03
[Epoch   19/ 100] [Batch    6/   9] Loss:  1.88e-03
[Epoch   19/ 100] [Batch    7/   9] Loss:  1.72e-03
[Epoch   19/ 100] [Batch    8/   9] Loss:  8.86e-04
[Epoch   19/ 100] [Batch    9/   9] Loss:  1.68e-03
[Epoch   20/ 100] [Batch    1/   9] Loss:  1.85e-03
[Epoch   20/ 100] [Batch    2/   9] Loss:  1.77e-03
[Epoch   20/ 100] [Batch    3/   9] Loss:  1.46e-03
[Epoch   20/ 100] [Batch    4/   9] Loss:  1.27e-03
[Epoch   20/ 100] [Batch    5/   9] Loss:  1.43e-03
[Epoch   20/ 100] [Batch    6/   9] Loss:  1.47e-03
[Epoch   20/ 100] [Batch    7/   9] Loss:  1.65e-03
[Epoch   20/ 100] [Batch    8/   9] Loss:  1.53e-03
[Epoch   20/ 100] [Batch    9/   9] Loss:  1.86e-03
[Epoch   21/ 100] [Batch    1/   9] Loss:  1.57e-03
[Epoch   21/ 100] [Batch    2/   9] Loss:  1.37e-03
[Epoch   21/ 100] [Batch    3/   9] Loss:  1.70e-03
[Epoch   21/ 100] [Batch    4/   9] Loss:  1.23e-03
[Epoch   21/ 100] [Batch    5/   9] Loss:  1.75e-03
[Epoch   21/ 100] [Batch    6/   9] Loss:  1.49e-03
[Epoch   21/ 100] [Batch    7/   9] Loss:  1.48e-03
[Epoch   21/ 100] [Batch    8/   9] Loss:  1.73e-03
[Epoch   21/ 100] [Batch    9/   9] Loss:  2.29e-03
[Epoch   22/ 100] [Batch    1/   9] Loss:  1.39e-03
[Epoch   22/ 100] [Batch    2/   9] Loss:  1.55e-03
[Epoch   22/ 100] [Batch    3/   9] Loss:  1.79e-03
[Epoch   22/ 100] [Batch    4/   9] Loss:  1.83e-03
[Epoch   22/ 100] [Batch    5/   9] Loss:  1.14e-03
[Epoch   22/ 100] [Batch    6/   9] Loss:  1.66e-03
[Epoch   22/ 100] [Batch    7/   9] Loss:  1.24e-03
[Epoch   22/ 100] [Batch    8/   9] Loss:  1.73e-03
[Epoch   22/ 100] [Batch    9/   9] Loss:  2.17e-03
[Epoch   23/ 100] [Batch    1/   9] Loss:  1.47e-03
[Epoch   23/ 100] [Batch    2/   9] Loss:  1.75e-03
[Epoch   23/ 100] [Batch    3/   9] Loss:  1.47e-03
[Epoch   23/ 100] [Batch    4/   9] Loss:  1.64e-03
[Epoch   23/ 100] [Batch    5/   9] Loss:  1.57e-03
[Epoch   23/ 100] [Batch    6/   9] Loss:  1.74e-03
[Epoch   23/ 100] [Batch    7/   9] Loss:  1.47e-03
[Epoch   23/ 100] [Batch    8/   9] Loss:  1.37e-03
[Epoch   23/ 100] [Batch    9/   9] Loss:  1.87e-03
[Epoch   24/ 100] [Batch    1/   9] Loss:  1.49e-03
[Epoch   24/ 100] [Batch    2/   9] Loss:  1.72e-03
[Epoch   24/ 100] [Batch    3/   9] Loss:  1.74e-03
[Epoch   24/ 100] [Batch    4/   9] Loss:  1.66e-03
[Epoch   24/ 100] [Batch    5/   9] Loss:  1.62e-03
[Epoch   24/ 100] [Batch    6/   9] Loss:  1.51e-03
[Epoch   24/ 100] [Batch    7/   9] Loss:  1.29e-03
[Epoch   24/ 100] [Batch    8/   9] Loss:  1.73e-03
[Epoch   24/ 100] [Batch    9/   9] Loss:  1.07e-03
[Epoch   25/ 100] [Batch    1/   9] Loss:  1.42e-03
[Epoch   25/ 100] [Batch    2/   9] Loss:  1.65e-03
[Epoch   25/ 100] [Batch    3/   9] Loss:  1.48e-03
[Epoch   25/ 100] [Batch    4/   9] Loss:  1.58e-03
[Epoch   25/ 100] [Batch    5/   9] Loss:  1.32e-03
[Epoch   25/ 100] [Batch    6/   9] Loss:  1.85e-03
[Epoch   25/ 100] [Batch    7/   9] Loss:  1.53e-03
[Epoch   25/ 100] [Batch    8/   9] Loss:  1.69e-03
[Epoch   25/ 100] [Batch    9/   9] Loss:  1.65e-03
[Epoch   26/ 100] [Batch    1/   9] Loss:  1.60e-03
[Epoch   26/ 100] [Batch    2/   9] Loss:  1.39e-03
[Epoch   26/ 100] [Batch    3/   9] Loss:  1.73e-03
[Epoch   26/ 100] [Batch    4/   9] Loss:  1.45e-03
[Epoch   26/ 100] [Batch    5/   9] Loss:  1.69e-03
[Epoch   26/ 100] [Batch    6/   9] Loss:  1.73e-03
[Epoch   26/ 100] [Batch    7/   9] Loss:  1.34e-03
[Epoch   26/ 100] [Batch    8/   9] Loss:  1.52e-03
[Epoch   26/ 100] [Batch    9/   9] Loss:  1.79e-03
[Epoch   27/ 100] [Batch    1/   9] Loss:  1.45e-03
[Epoch   27/ 100] [Batch    2/   9] Loss:  1.30e-03
[Epoch   27/ 100] [Batch    3/   9] Loss:  1.53e-03
[Epoch   27/ 100] [Batch    4/   9] Loss:  1.87e-03
[Epoch   27/ 100] [Batch    5/   9] Loss:  1.30e-03
[Epoch   27/ 100] [Batch    6/   9] Loss:  1.78e-03
[Epoch   27/ 100] [Batch    7/   9] Loss:  1.55e-03
[Epoch   27/ 100] [Batch    8/   9] Loss:  1.82e-03
[Epoch   27/ 100] [Batch    9/   9] Loss:  1.42e-03
[Epoch   28/ 100] [Batch    1/   9] Loss:  1.59e-03
[Epoch   28/ 100] [Batch    2/   9] Loss:  1.73e-03
[Epoch   28/ 100] [Batch    3/   9] Loss:  1.48e-03
[Epoch   28/ 100] [Batch    4/   9] Loss:  1.40e-03
[Epoch   28/ 100] [Batch    5/   9] Loss:  1.47e-03
[Epoch   28/ 100] [Batch    6/   9] Loss:  1.90e-03
[Epoch   28/ 100] [Batch    7/   9] Loss:  1.31e-03
[Epoch   28/ 100] [Batch    8/   9] Loss:  1.69e-03
[Epoch   28/ 100] [Batch    9/   9] Loss:  1.58e-03
[Epoch   29/ 100] [Batch    1/   9] Loss:  1.59e-03
[Epoch   29/ 100] [Batch    2/   9] Loss:  1.31e-03
[Epoch   29/ 100] [Batch    3/   9] Loss:  1.41e-03
[Epoch   29/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   29/ 100] [Batch    5/   9] Loss:  1.58e-03
[Epoch   29/ 100] [Batch    6/   9] Loss:  1.57e-03
[Epoch   29/ 100] [Batch    7/   9] Loss:  1.94e-03
[Epoch   29/ 100] [Batch    8/   9] Loss:  1.67e-03
[Epoch   29/ 100] [Batch    9/   9] Loss:  1.54e-03
[Epoch   30/ 100] [Batch    1/   9] Loss:  1.54e-03
[Epoch   30/ 100] [Batch    2/   9] Loss:  1.37e-03
[Epoch   30/ 100] [Batch    3/   9] Loss:  1.65e-03
[Epoch   30/ 100] [Batch    4/   9] Loss:  1.46e-03
[Epoch   30/ 100] [Batch    5/   9] Loss:  1.43e-03
[Epoch   30/ 100] [Batch    6/   9] Loss:  1.51e-03
[Epoch   30/ 100] [Batch    7/   9] Loss:  1.60e-03
[Epoch   30/ 100] [Batch    8/   9] Loss:  1.69e-03
[Epoch   30/ 100] [Batch    9/   9] Loss:  2.47e-03
[Epoch   31/ 100] [Batch    1/   9] Loss:  1.25e-03
[Epoch   31/ 100] [Batch    2/   9] Loss:  1.48e-03
[Epoch   31/ 100] [Batch    3/   9] Loss:  1.39e-03
[Epoch   31/ 100] [Batch    4/   9] Loss:  1.90e-03
[Epoch   31/ 100] [Batch    5/   9] Loss:  1.59e-03
[Epoch   31/ 100] [Batch    6/   9] Loss:  1.91e-03
[Epoch   31/ 100] [Batch    7/   9] Loss:  1.48e-03
[Epoch   31/ 100] [Batch    8/   9] Loss:  1.71e-03
[Epoch   31/ 100] [Batch    9/   9] Loss:  1.21e-03
[Epoch   32/ 100] [Batch    1/   9] Loss:  1.59e-03
[Epoch   32/ 100] [Batch    2/   9] Loss:  1.41e-03
[Epoch   32/ 100] [Batch    3/   9] Loss:  1.77e-03
[Epoch   32/ 100] [Batch    4/   9] Loss:  1.16e-03
[Epoch   32/ 100] [Batch    5/   9] Loss:  1.13e-03
[Epoch   32/ 100] [Batch    6/   9] Loss:  1.81e-03
[Epoch   32/ 100] [Batch    7/   9] Loss:  1.98e-03
[Epoch   32/ 100] [Batch    8/   9] Loss:  1.64e-03
[Epoch   32/ 100] [Batch    9/   9] Loss:  1.72e-03
[Epoch   33/ 100] [Batch    1/   9] Loss:  1.33e-03
[Epoch   33/ 100] [Batch    2/   9] Loss:  1.50e-03
[Epoch   33/ 100] [Batch    3/   9] Loss:  1.38e-03
[Epoch   33/ 100] [Batch    4/   9] Loss:  1.61e-03
[Epoch   33/ 100] [Batch    5/   9] Loss:  2.11e-03
[Epoch   33/ 100] [Batch    6/   9] Loss:  1.44e-03
[Epoch   33/ 100] [Batch    7/   9] Loss:  1.84e-03
[Epoch   33/ 100] [Batch    8/   9] Loss:  1.39e-03
[Epoch   33/ 100] [Batch    9/   9] Loss:  1.48e-03
[Epoch   34/ 100] [Batch    1/   9] Loss:  1.64e-03
[Epoch   34/ 100] [Batch    2/   9] Loss:  1.32e-03
[Epoch   34/ 100] [Batch    3/   9] Loss:  1.66e-03
[Epoch   34/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   34/ 100] [Batch    5/   9] Loss:  1.83e-03
[Epoch   34/ 100] [Batch    6/   9] Loss:  1.35e-03
[Epoch   34/ 100] [Batch    7/   9] Loss:  1.87e-03
[Epoch   34/ 100] [Batch    8/   9] Loss:  1.48e-03
[Epoch   34/ 100] [Batch    9/   9] Loss:  1.12e-03
[Epoch   35/ 100] [Batch    1/   9] Loss:  1.24e-03
[Epoch   35/ 100] [Batch    2/   9] Loss:  1.71e-03
[Epoch   35/ 100] [Batch    3/   9] Loss:  1.93e-03
[Epoch   35/ 100] [Batch    4/   9] Loss:  1.60e-03
[Epoch   35/ 100] [Batch    5/   9] Loss:  1.38e-03
[Epoch   35/ 100] [Batch    6/   9] Loss:  1.40e-03
[Epoch   35/ 100] [Batch    7/   9] Loss:  1.57e-03
[Epoch   35/ 100] [Batch    8/   9] Loss:  1.67e-03
[Epoch   35/ 100] [Batch    9/   9] Loss:  1.77e-03
[Epoch   36/ 100] [Batch    1/   9] Loss:  1.52e-03
[Epoch   36/ 100] [Batch    2/   9] Loss:  1.48e-03
[Epoch   36/ 100] [Batch    3/   9] Loss:  1.35e-03
[Epoch   36/ 100] [Batch    4/   9] Loss:  1.61e-03
[Epoch   36/ 100] [Batch    5/   9] Loss:  1.52e-03
[Epoch   36/ 100] [Batch    6/   9] Loss:  1.64e-03
[Epoch   36/ 100] [Batch    7/   9] Loss:  1.81e-03
[Epoch   36/ 100] [Batch    8/   9] Loss:  1.75e-03
[Epoch   36/ 100] [Batch    9/   9] Loss:  1.18e-03
[Epoch   37/ 100] [Batch    1/   9] Loss:  1.74e-03
[Epoch   37/ 100] [Batch    2/   9] Loss:  1.65e-03
[Epoch   37/ 100] [Batch    3/   9] Loss:  1.45e-03
[Epoch   37/ 100] [Batch    4/   9] Loss:  1.64e-03
[Epoch   37/ 100] [Batch    5/   9] Loss:  1.70e-03
[Epoch   37/ 100] [Batch    6/   9] Loss:  1.52e-03
[Epoch   37/ 100] [Batch    7/   9] Loss:  1.35e-03
[Epoch   37/ 100] [Batch    8/   9] Loss:  1.44e-03
[Epoch   37/ 100] [Batch    9/   9] Loss:  1.72e-03
[Epoch   38/ 100] [Batch    1/   9] Loss:  1.51e-03
[Epoch   38/ 100] [Batch    2/   9] Loss:  1.49e-03
[Epoch   38/ 100] [Batch    3/   9] Loss:  1.30e-03
[Epoch   38/ 100] [Batch    4/   9] Loss:  1.53e-03
[Epoch   38/ 100] [Batch    5/   9] Loss:  1.86e-03
[Epoch   38/ 100] [Batch    6/   9] Loss:  1.71e-03
[Epoch   38/ 100] [Batch    7/   9] Loss:  1.45e-03
[Epoch   38/ 100] [Batch    8/   9] Loss:  1.44e-03
[Epoch   38/ 100] [Batch    9/   9] Loss:  2.34e-03
[Epoch   39/ 100] [Batch    1/   9] Loss:  1.55e-03
[Epoch   39/ 100] [Batch    2/   9] Loss:  1.84e-03
[Epoch   39/ 100] [Batch    3/   9] Loss:  1.75e-03
[Epoch   39/ 100] [Batch    4/   9] Loss:  1.67e-03
[Epoch   39/ 100] [Batch    5/   9] Loss:  1.19e-03
[Epoch   39/ 100] [Batch    6/   9] Loss:  1.28e-03
[Epoch   39/ 100] [Batch    7/   9] Loss:  1.78e-03
[Epoch   39/ 100] [Batch    8/   9] Loss:  1.54e-03
[Epoch   39/ 100] [Batch    9/   9] Loss:  1.34e-03
[Epoch   40/ 100] [Batch    1/   9] Loss:  1.80e-03
[Epoch   40/ 100] [Batch    2/   9] Loss:  1.34e-03
[Epoch   40/ 100] [Batch    3/   9] Loss:  1.67e-03
[Epoch   40/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   40/ 100] [Batch    5/   9] Loss:  1.67e-03
[Epoch   40/ 100] [Batch    6/   9] Loss:  1.28e-03
[Epoch   40/ 100] [Batch    7/   9] Loss:  1.81e-03
[Epoch   40/ 100] [Batch    8/   9] Loss:  1.47e-03
[Epoch   40/ 100] [Batch    9/   9] Loss:  1.37e-03
[Epoch   41/ 100] [Batch    1/   9] Loss:  1.63e-03
[Epoch   41/ 100] [Batch    2/   9] Loss:  1.53e-03
[Epoch   41/ 100] [Batch    3/   9] Loss:  1.58e-03
[Epoch   41/ 100] [Batch    4/   9] Loss:  1.37e-03
[Epoch   41/ 100] [Batch    5/   9] Loss:  1.48e-03
[Epoch   41/ 100] [Batch    6/   9] Loss:  1.79e-03
[Epoch   41/ 100] [Batch    7/   9] Loss:  1.39e-03
[Epoch   41/ 100] [Batch    8/   9] Loss:  1.74e-03
[Epoch   41/ 100] [Batch    9/   9] Loss:  1.61e-03
[Epoch   42/ 100] [Batch    1/   9] Loss:  1.45e-03
[Epoch   42/ 100] [Batch    2/   9] Loss:  1.61e-03
[Epoch   42/ 100] [Batch    3/   9] Loss:  1.74e-03
[Epoch   42/ 100] [Batch    4/   9] Loss:  1.53e-03
[Epoch   42/ 100] [Batch    5/   9] Loss:  1.57e-03
[Epoch   42/ 100] [Batch    6/   9] Loss:  1.47e-03
[Epoch   42/ 100] [Batch    7/   9] Loss:  1.61e-03
[Epoch   42/ 100] [Batch    8/   9] Loss:  1.43e-03
[Epoch   42/ 100] [Batch    9/   9] Loss:  1.90e-03
[Epoch   43/ 100] [Batch    1/   9] Loss:  1.20e-03
[Epoch   43/ 100] [Batch    2/   9] Loss:  1.73e-03
[Epoch   43/ 100] [Batch    3/   9] Loss:  1.44e-03
[Epoch   43/ 100] [Batch    4/   9] Loss:  1.71e-03
[Epoch   43/ 100] [Batch    5/   9] Loss:  1.61e-03
[Epoch   43/ 100] [Batch    6/   9] Loss:  1.47e-03
[Epoch   43/ 100] [Batch    7/   9] Loss:  1.58e-03
[Epoch   43/ 100] [Batch    8/   9] Loss:  1.88e-03
[Epoch   43/ 100] [Batch    9/   9] Loss:  1.39e-03
[Epoch   44/ 100] [Batch    1/   9] Loss:  1.35e-03
[Epoch   44/ 100] [Batch    2/   9] Loss:  1.32e-03
[Epoch   44/ 100] [Batch    3/   9] Loss:  1.56e-03
[Epoch   44/ 100] [Batch    4/   9] Loss:  1.25e-03
[Epoch   44/ 100] [Batch    5/   9] Loss:  1.61e-03
[Epoch   44/ 100] [Batch    6/   9] Loss:  1.78e-03
[Epoch   44/ 100] [Batch    7/   9] Loss:  1.93e-03
[Epoch   44/ 100] [Batch    8/   9] Loss:  1.95e-03
[Epoch   44/ 100] [Batch    9/   9] Loss:  1.07e-03
[Epoch   45/ 100] [Batch    1/   9] Loss:  1.68e-03
[Epoch   45/ 100] [Batch    2/   9] Loss:  1.49e-03
[Epoch   45/ 100] [Batch    3/   9] Loss:  1.60e-03
[Epoch   45/ 100] [Batch    4/   9] Loss:  1.22e-03
[Epoch   45/ 100] [Batch    5/   9] Loss:  1.52e-03
[Epoch   45/ 100] [Batch    6/   9] Loss:  1.84e-03
[Epoch   45/ 100] [Batch    7/   9] Loss:  1.57e-03
[Epoch   45/ 100] [Batch    8/   9] Loss:  1.71e-03
[Epoch   45/ 100] [Batch    9/   9] Loss:  1.39e-03
[Epoch   46/ 100] [Batch    1/   9] Loss:  1.54e-03
[Epoch   46/ 100] [Batch    2/   9] Loss:  1.53e-03
[Epoch   46/ 100] [Batch    3/   9] Loss:  1.53e-03
[Epoch   46/ 100] [Batch    4/   9] Loss:  1.32e-03
[Epoch   46/ 100] [Batch    5/   9] Loss:  1.36e-03
[Epoch   46/ 100] [Batch    6/   9] Loss:  1.67e-03
[Epoch   46/ 100] [Batch    7/   9] Loss:  1.61e-03
[Epoch   46/ 100] [Batch    8/   9] Loss:  2.04e-03
[Epoch   46/ 100] [Batch    9/   9] Loss:  1.38e-03
[Epoch   47/ 100] [Batch    1/   9] Loss:  1.18e-03
[Epoch   47/ 100] [Batch    2/   9] Loss:  1.41e-03
[Epoch   47/ 100] [Batch    3/   9] Loss:  1.67e-03
[Epoch   47/ 100] [Batch    4/   9] Loss:  1.70e-03
[Epoch   47/ 100] [Batch    5/   9] Loss:  1.87e-03
[Epoch   47/ 100] [Batch    6/   9] Loss:  1.70e-03
[Epoch   47/ 100] [Batch    7/   9] Loss:  1.42e-03
[Epoch   47/ 100] [Batch    8/   9] Loss:  1.76e-03
[Epoch   47/ 100] [Batch    9/   9] Loss:  1.14e-03
[Epoch   48/ 100] [Batch    1/   9] Loss:  1.75e-03
[Epoch   48/ 100] [Batch    2/   9] Loss:  1.78e-03
[Epoch   48/ 100] [Batch    3/   9] Loss:  1.63e-03
[Epoch   48/ 100] [Batch    4/   9] Loss:  1.52e-03
[Epoch   48/ 100] [Batch    5/   9] Loss:  1.21e-03
[Epoch   48/ 100] [Batch    6/   9] Loss:  1.77e-03
[Epoch   48/ 100] [Batch    7/   9] Loss:  1.39e-03
[Epoch   48/ 100] [Batch    8/   9] Loss:  1.55e-03
[Epoch   48/ 100] [Batch    9/   9] Loss:  1.41e-03
[Epoch   49/ 100] [Batch    1/   9] Loss:  1.59e-03
[Epoch   49/ 100] [Batch    2/   9] Loss:  1.54e-03
[Epoch   49/ 100] [Batch    3/   9] Loss:  1.84e-03
[Epoch   49/ 100] [Batch    4/   9] Loss:  1.68e-03
[Epoch   49/ 100] [Batch    5/   9] Loss:  1.58e-03
[Epoch   49/ 100] [Batch    6/   9] Loss:  1.59e-03
[Epoch   49/ 100] [Batch    7/   9] Loss:  1.62e-03
[Epoch   49/ 100] [Batch    8/   9] Loss:  1.39e-03
[Epoch   49/ 100] [Batch    9/   9] Loss:  7.44e-04
[Epoch   50/ 100] [Batch    1/   9] Loss:  1.31e-03
[Epoch   50/ 100] [Batch    2/   9] Loss:  1.77e-03
[Epoch   50/ 100] [Batch    3/   9] Loss:  1.83e-03
[Epoch   50/ 100] [Batch    4/   9] Loss:  1.11e-03
[Epoch   50/ 100] [Batch    5/   9] Loss:  1.72e-03
[Epoch   50/ 100] [Batch    6/   9] Loss:  1.66e-03
[Epoch   50/ 100] [Batch    7/   9] Loss:  1.65e-03
[Epoch   50/ 100] [Batch    8/   9] Loss:  1.36e-03
[Epoch   50/ 100] [Batch    9/   9] Loss:  1.95e-03
[Epoch   51/ 100] [Batch    1/   9] Loss:  1.27e-03
[Epoch   51/ 100] [Batch    2/   9] Loss:  1.72e-03
[Epoch   51/ 100] [Batch    3/   9] Loss:  1.54e-03
[Epoch   51/ 100] [Batch    4/   9] Loss:  1.57e-03
[Epoch   51/ 100] [Batch    5/   9] Loss:  1.44e-03
[Epoch   51/ 100] [Batch    6/   9] Loss:  1.53e-03
[Epoch   51/ 100] [Batch    7/   9] Loss:  2.06e-03
[Epoch   51/ 100] [Batch    8/   9] Loss:  1.59e-03
[Epoch   51/ 100] [Batch    9/   9] Loss:  1.47e-03
[Epoch   52/ 100] [Batch    1/   9] Loss:  1.56e-03
[Epoch   52/ 100] [Batch    2/   9] Loss:  1.46e-03
[Epoch   52/ 100] [Batch    3/   9] Loss:  1.76e-03
[Epoch   52/ 100] [Batch    4/   9] Loss:  1.39e-03
[Epoch   52/ 100] [Batch    5/   9] Loss:  1.83e-03
[Epoch   52/ 100] [Batch    6/   9] Loss:  1.44e-03
[Epoch   52/ 100] [Batch    7/   9] Loss:  1.47e-03
[Epoch   52/ 100] [Batch    8/   9] Loss:  1.69e-03
[Epoch   52/ 100] [Batch    9/   9] Loss:  1.61e-03
[Epoch   53/ 100] [Batch    1/   9] Loss:  1.40e-03
[Epoch   53/ 100] [Batch    2/   9] Loss:  1.76e-03
[Epoch   53/ 100] [Batch    3/   9] Loss:  1.24e-03
[Epoch   53/ 100] [Batch    4/   9] Loss:  1.65e-03
[Epoch   53/ 100] [Batch    5/   9] Loss:  1.68e-03
[Epoch   53/ 100] [Batch    6/   9] Loss:  1.55e-03
[Epoch   53/ 100] [Batch    7/   9] Loss:  1.75e-03
[Epoch   53/ 100] [Batch    8/   9] Loss:  1.64e-03
[Epoch   53/ 100] [Batch    9/   9] Loss:  1.33e-03
[Epoch   54/ 100] [Batch    1/   9] Loss:  1.60e-03
[Epoch   54/ 100] [Batch    2/   9] Loss:  1.80e-03
[Epoch   54/ 100] [Batch    3/   9] Loss:  1.38e-03
[Epoch   54/ 100] [Batch    4/   9] Loss:  1.67e-03
[Epoch   54/ 100] [Batch    5/   9] Loss:  1.85e-03
[Epoch   54/ 100] [Batch    6/   9] Loss:  1.27e-03
[Epoch   54/ 100] [Batch    7/   9] Loss:  1.28e-03
[Epoch   54/ 100] [Batch    8/   9] Loss:  1.64e-03
[Epoch   54/ 100] [Batch    9/   9] Loss:  1.75e-03
[Epoch   55/ 100] [Batch    1/   9] Loss:  1.51e-03
[Epoch   55/ 100] [Batch    2/   9] Loss:  1.58e-03
[Epoch   55/ 100] [Batch    3/   9] Loss:  1.57e-03
[Epoch   55/ 100] [Batch    4/   9] Loss:  1.64e-03
[Epoch   55/ 100] [Batch    5/   9] Loss:  1.35e-03
[Epoch   55/ 100] [Batch    6/   9] Loss:  1.65e-03
[Epoch   55/ 100] [Batch    7/   9] Loss:  1.74e-03
[Epoch   55/ 100] [Batch    8/   9] Loss:  1.42e-03
[Epoch   55/ 100] [Batch    9/   9] Loss:  1.76e-03
[Epoch   56/ 100] [Batch    1/   9] Loss:  1.76e-03
[Epoch   56/ 100] [Batch    2/   9] Loss:  1.43e-03
[Epoch   56/ 100] [Batch    3/   9] Loss:  1.42e-03
[Epoch   56/ 100] [Batch    4/   9] Loss:  1.70e-03
[Epoch   56/ 100] [Batch    5/   9] Loss:  1.58e-03
[Epoch   56/ 100] [Batch    6/   9] Loss:  1.39e-03
[Epoch   56/ 100] [Batch    7/   9] Loss:  1.82e-03
[Epoch   56/ 100] [Batch    8/   9] Loss:  1.49e-03
[Epoch   56/ 100] [Batch    9/   9] Loss:  1.46e-03
[Epoch   57/ 100] [Batch    1/   9] Loss:  1.64e-03
[Epoch   57/ 100] [Batch    2/   9] Loss:  1.37e-03
[Epoch   57/ 100] [Batch    3/   9] Loss:  1.64e-03
[Epoch   57/ 100] [Batch    4/   9] Loss:  1.43e-03
[Epoch   57/ 100] [Batch    5/   9] Loss:  1.71e-03
[Epoch   57/ 100] [Batch    6/   9] Loss:  1.59e-03
[Epoch   57/ 100] [Batch    7/   9] Loss:  1.69e-03
[Epoch   57/ 100] [Batch    8/   9] Loss:  1.48e-03
[Epoch   57/ 100] [Batch    9/   9] Loss:  1.88e-03
[Epoch   58/ 100] [Batch    1/   9] Loss:  1.71e-03
[Epoch   58/ 100] [Batch    2/   9] Loss:  1.33e-03
[Epoch   58/ 100] [Batch    3/   9] Loss:  1.46e-03
[Epoch   58/ 100] [Batch    4/   9] Loss:  1.44e-03
[Epoch   58/ 100] [Batch    5/   9] Loss:  2.12e-03
[Epoch   58/ 100] [Batch    6/   9] Loss:  1.57e-03
[Epoch   58/ 100] [Batch    7/   9] Loss:  1.39e-03
[Epoch   58/ 100] [Batch    8/   9] Loss:  1.64e-03
[Epoch   58/ 100] [Batch    9/   9] Loss:  1.38e-03
[Epoch   59/ 100] [Batch    1/   9] Loss:  2.05e-03
[Epoch   59/ 100] [Batch    2/   9] Loss:  1.88e-03
[Epoch   59/ 100] [Batch    3/   9] Loss:  1.76e-03
[Epoch   59/ 100] [Batch    4/   9] Loss:  1.68e-03
[Epoch   59/ 100] [Batch    5/   9] Loss:  1.41e-03
[Epoch   59/ 100] [Batch    6/   9] Loss:  1.30e-03
[Epoch   59/ 100] [Batch    7/   9] Loss:  1.31e-03
[Epoch   59/ 100] [Batch    8/   9] Loss:  1.33e-03
[Epoch   59/ 100] [Batch    9/   9] Loss:  1.09e-03
[Epoch   60/ 100] [Batch    1/   9] Loss:  1.66e-03
[Epoch   60/ 100] [Batch    2/   9] Loss:  1.77e-03
[Epoch   60/ 100] [Batch    3/   9] Loss:  1.55e-03
[Epoch   60/ 100] [Batch    4/   9] Loss:  1.40e-03
[Epoch   60/ 100] [Batch    5/   9] Loss:  1.39e-03
[Epoch   60/ 100] [Batch    6/   9] Loss:  1.71e-03
[Epoch   60/ 100] [Batch    7/   9] Loss:  1.65e-03
[Epoch   60/ 100] [Batch    8/   9] Loss:  1.56e-03
[Epoch   60/ 100] [Batch    9/   9] Loss:  1.06e-03
[Epoch   61/ 100] [Batch    1/   9] Loss:  1.44e-03
[Epoch   61/ 100] [Batch    2/   9] Loss:  1.77e-03
[Epoch   61/ 100] [Batch    3/   9] Loss:  1.71e-03
[Epoch   61/ 100] [Batch    4/   9] Loss:  1.45e-03
[Epoch   61/ 100] [Batch    5/   9] Loss:  1.48e-03
[Epoch   61/ 100] [Batch    6/   9] Loss:  1.69e-03
[Epoch   61/ 100] [Batch    7/   9] Loss:  1.46e-03
[Epoch   61/ 100] [Batch    8/   9] Loss:  1.33e-03
[Epoch   61/ 100] [Batch    9/   9] Loss:  2.23e-03
[Epoch   62/ 100] [Batch    1/   9] Loss:  1.19e-03
[Epoch   62/ 100] [Batch    2/   9] Loss:  1.54e-03
[Epoch   62/ 100] [Batch    3/   9] Loss:  1.71e-03
[Epoch   62/ 100] [Batch    4/   9] Loss:  1.38e-03
[Epoch   62/ 100] [Batch    5/   9] Loss:  1.92e-03
[Epoch   62/ 100] [Batch    6/   9] Loss:  1.84e-03
[Epoch   62/ 100] [Batch    7/   9] Loss:  1.50e-03
[Epoch   62/ 100] [Batch    8/   9] Loss:  1.68e-03
[Epoch   62/ 100] [Batch    9/   9] Loss:  1.17e-03
[Epoch   63/ 100] [Batch    1/   9] Loss:  1.82e-03
[Epoch   63/ 100] [Batch    2/   9] Loss:  1.40e-03
[Epoch   63/ 100] [Batch    3/   9] Loss:  1.71e-03
[Epoch   63/ 100] [Batch    4/   9] Loss:  1.87e-03
[Epoch   63/ 100] [Batch    5/   9] Loss:  1.46e-03
[Epoch   63/ 100] [Batch    6/   9] Loss:  1.46e-03
[Epoch   63/ 100] [Batch    7/   9] Loss:  1.64e-03
[Epoch   63/ 100] [Batch    8/   9] Loss:  1.35e-03
[Epoch   63/ 100] [Batch    9/   9] Loss:  1.41e-03
[Epoch   64/ 100] [Batch    1/   9] Loss:  1.70e-03
[Epoch   64/ 100] [Batch    2/   9] Loss:  1.38e-03
[Epoch   64/ 100] [Batch    3/   9] Loss:  1.81e-03
[Epoch   64/ 100] [Batch    4/   9] Loss:  1.73e-03
[Epoch   64/ 100] [Batch    5/   9] Loss:  1.36e-03
[Epoch   64/ 100] [Batch    6/   9] Loss:  1.52e-03
[Epoch   64/ 100] [Batch    7/   9] Loss:  1.64e-03
[Epoch   64/ 100] [Batch    8/   9] Loss:  1.58e-03
[Epoch   64/ 100] [Batch    9/   9] Loss:  1.18e-03
[Epoch   65/ 100] [Batch    1/   9] Loss:  1.84e-03
[Epoch   65/ 100] [Batch    2/   9] Loss:  1.34e-03
[Epoch   65/ 100] [Batch    3/   9] Loss:  1.58e-03
[Epoch   65/ 100] [Batch    4/   9] Loss:  1.47e-03
[Epoch   65/ 100] [Batch    5/   9] Loss:  1.63e-03
[Epoch   65/ 100] [Batch    6/   9] Loss:  1.54e-03
[Epoch   65/ 100] [Batch    7/   9] Loss:  1.67e-03
[Epoch   65/ 100] [Batch    8/   9] Loss:  1.42e-03
[Epoch   65/ 100] [Batch    9/   9] Loss:  2.09e-03
[Epoch   66/ 100] [Batch    1/   9] Loss:  1.39e-03
[Epoch   66/ 100] [Batch    2/   9] Loss:  1.55e-03
[Epoch   66/ 100] [Batch    3/   9] Loss:  1.50e-03
[Epoch   66/ 100] [Batch    4/   9] Loss:  1.62e-03
[Epoch   66/ 100] [Batch    5/   9] Loss:  1.66e-03
[Epoch   66/ 100] [Batch    6/   9] Loss:  1.31e-03
[Epoch   66/ 100] [Batch    7/   9] Loss:  1.92e-03
[Epoch   66/ 100] [Batch    8/   9] Loss:  1.51e-03
[Epoch   66/ 100] [Batch    9/   9] Loss:  2.04e-03
[Epoch   67/ 100] [Batch    1/   9] Loss:  1.49e-03
[Epoch   67/ 100] [Batch    2/   9] Loss:  1.94e-03
[Epoch   67/ 100] [Batch    3/   9] Loss:  1.62e-03
[Epoch   67/ 100] [Batch    4/   9] Loss:  1.60e-03
[Epoch   67/ 100] [Batch    5/   9] Loss:  1.48e-03
[Epoch   67/ 100] [Batch    6/   9] Loss:  1.46e-03
[Epoch   67/ 100] [Batch    7/   9] Loss:  1.66e-03
[Epoch   67/ 100] [Batch    8/   9] Loss:  1.55e-03
[Epoch   67/ 100] [Batch    9/   9] Loss:  9.59e-04
[Epoch   68/ 100] [Batch    1/   9] Loss:  1.38e-03
[Epoch   68/ 100] [Batch    2/   9] Loss:  1.58e-03
[Epoch   68/ 100] [Batch    3/   9] Loss:  1.54e-03
[Epoch   68/ 100] [Batch    4/   9] Loss:  1.36e-03
[Epoch   68/ 100] [Batch    5/   9] Loss:  1.76e-03
[Epoch   68/ 100] [Batch    6/   9] Loss:  1.68e-03
[Epoch   68/ 100] [Batch    7/   9] Loss:  1.54e-03
[Epoch   68/ 100] [Batch    8/   9] Loss:  1.64e-03
[Epoch   68/ 100] [Batch    9/   9] Loss:  1.87e-03
[Epoch   69/ 100] [Batch    1/   9] Loss:  1.91e-03
[Epoch   69/ 100] [Batch    2/   9] Loss:  1.62e-03
[Epoch   69/ 100] [Batch    3/   9] Loss:  1.63e-03
[Epoch   69/ 100] [Batch    4/   9] Loss:  1.40e-03
[Epoch   69/ 100] [Batch    5/   9] Loss:  1.33e-03
[Epoch   69/ 100] [Batch    6/   9] Loss:  1.65e-03
[Epoch   69/ 100] [Batch    7/   9] Loss:  1.41e-03
[Epoch   69/ 100] [Batch    8/   9] Loss:  1.43e-03
[Epoch   69/ 100] [Batch    9/   9] Loss:  2.12e-03
[Epoch   70/ 100] [Batch    1/   9] Loss:  1.82e-03
[Epoch   70/ 100] [Batch    2/   9] Loss:  1.73e-03
[Epoch   70/ 100] [Batch    3/   9] Loss:  1.16e-03
[Epoch   70/ 100] [Batch    4/   9] Loss:  1.70e-03
[Epoch   70/ 100] [Batch    5/   9] Loss:  1.52e-03
[Epoch   70/ 100] [Batch    6/   9] Loss:  1.59e-03
[Epoch   70/ 100] [Batch    7/   9] Loss:  1.63e-03
[Epoch   70/ 100] [Batch    8/   9] Loss:  1.39e-03
[Epoch   70/ 100] [Batch    9/   9] Loss:  1.73e-03
[Epoch   71/ 100] [Batch    1/   9] Loss:  1.65e-03
[Epoch   71/ 100] [Batch    2/   9] Loss:  1.85e-03
[Epoch   71/ 100] [Batch    3/   9] Loss:  1.41e-03
[Epoch   71/ 100] [Batch    4/   9] Loss:  1.37e-03
[Epoch   71/ 100] [Batch    5/   9] Loss:  1.81e-03
[Epoch   71/ 100] [Batch    6/   9] Loss:  1.71e-03
[Epoch   71/ 100] [Batch    7/   9] Loss:  1.48e-03
[Epoch   71/ 100] [Batch    8/   9] Loss:  1.29e-03
[Epoch   71/ 100] [Batch    9/   9] Loss:  1.49e-03
[Epoch   72/ 100] [Batch    1/   9] Loss:  1.53e-03
[Epoch   72/ 100] [Batch    2/   9] Loss:  1.63e-03
[Epoch   72/ 100] [Batch    3/   9] Loss:  1.37e-03
[Epoch   72/ 100] [Batch    4/   9] Loss:  1.84e-03
[Epoch   72/ 100] [Batch    5/   9] Loss:  1.19e-03
[Epoch   72/ 100] [Batch    6/   9] Loss:  1.93e-03
[Epoch   72/ 100] [Batch    7/   9] Loss:  1.65e-03
[Epoch   72/ 100] [Batch    8/   9] Loss:  1.52e-03
[Epoch   72/ 100] [Batch    9/   9] Loss:  1.37e-03
[Epoch   73/ 100] [Batch    1/   9] Loss:  1.40e-03
[Epoch   73/ 100] [Batch    2/   9] Loss:  1.20e-03
[Epoch   73/ 100] [Batch    3/   9] Loss:  1.62e-03
[Epoch   73/ 100] [Batch    4/   9] Loss:  1.55e-03
[Epoch   73/ 100] [Batch    5/   9] Loss:  1.27e-03
[Epoch   73/ 100] [Batch    6/   9] Loss:  1.72e-03
[Epoch   73/ 100] [Batch    7/   9] Loss:  1.95e-03
[Epoch   73/ 100] [Batch    8/   9] Loss:  1.77e-03
[Epoch   73/ 100] [Batch    9/   9] Loss:  1.94e-03
[Epoch   74/ 100] [Batch    1/   9] Loss:  1.71e-03
[Epoch   74/ 100] [Batch    2/   9] Loss:  1.52e-03
[Epoch   74/ 100] [Batch    3/   9] Loss:  1.55e-03
[Epoch   74/ 100] [Batch    4/   9] Loss:  1.76e-03
[Epoch   74/ 100] [Batch    5/   9] Loss:  1.65e-03
[Epoch   74/ 100] [Batch    6/   9] Loss:  1.60e-03
[Epoch   74/ 100] [Batch    7/   9] Loss:  1.54e-03
[Epoch   74/ 100] [Batch    8/   9] Loss:  1.30e-03
[Epoch   74/ 100] [Batch    9/   9] Loss:  1.47e-03
[Epoch   75/ 100] [Batch    1/   9] Loss:  1.66e-03
[Epoch   75/ 100] [Batch    2/   9] Loss:  1.76e-03
[Epoch   75/ 100] [Batch    3/   9] Loss:  1.64e-03
[Epoch   75/ 100] [Batch    4/   9] Loss:  1.82e-03
[Epoch   75/ 100] [Batch    5/   9] Loss:  1.11e-03
[Epoch   75/ 100] [Batch    6/   9] Loss:  1.28e-03
[Epoch   75/ 100] [Batch    7/   9] Loss:  1.52e-03
[Epoch   75/ 100] [Batch    8/   9] Loss:  1.73e-03
[Epoch   75/ 100] [Batch    9/   9] Loss:  1.62e-03
[Epoch   76/ 100] [Batch    1/   9] Loss:  1.55e-03
[Epoch   76/ 100] [Batch    2/   9] Loss:  1.44e-03
[Epoch   76/ 100] [Batch    3/   9] Loss:  1.78e-03
[Epoch   76/ 100] [Batch    4/   9] Loss:  1.41e-03
[Epoch   76/ 100] [Batch    5/   9] Loss:  1.65e-03
[Epoch   76/ 100] [Batch    6/   9] Loss:  1.77e-03
[Epoch   76/ 100] [Batch    7/   9] Loss:  1.48e-03
[Epoch   76/ 100] [Batch    8/   9] Loss:  1.48e-03
[Epoch   76/ 100] [Batch    9/   9] Loss:  1.62e-03
[Epoch   77/ 100] [Batch    1/   9] Loss:  1.39e-03
[Epoch   77/ 100] [Batch    2/   9] Loss:  1.18e-03
[Epoch   77/ 100] [Batch    3/   9] Loss:  1.91e-03
[Epoch   77/ 100] [Batch    4/   9] Loss:  1.27e-03
[Epoch   77/ 100] [Batch    5/   9] Loss:  1.44e-03
[Epoch   77/ 100] [Batch    6/   9] Loss:  1.85e-03
[Epoch   77/ 100] [Batch    7/   9] Loss:  1.78e-03
[Epoch   77/ 100] [Batch    8/   9] Loss:  1.82e-03
[Epoch   77/ 100] [Batch    9/   9] Loss:  1.53e-03
[Epoch   78/ 100] [Batch    1/   9] Loss:  1.72e-03
[Epoch   78/ 100] [Batch    2/   9] Loss:  1.53e-03
[Epoch   78/ 100] [Batch    3/   9] Loss:  1.66e-03
[Epoch   78/ 100] [Batch    4/   9] Loss:  1.59e-03
[Epoch   78/ 100] [Batch    5/   9] Loss:  1.63e-03
[Epoch   78/ 100] [Batch    6/   9] Loss:  1.54e-03
[Epoch   78/ 100] [Batch    7/   9] Loss:  1.47e-03
[Epoch   78/ 100] [Batch    8/   9] Loss:  1.44e-03
[Epoch   78/ 100] [Batch    9/   9] Loss:  1.47e-03
[Epoch   79/ 100] [Batch    1/   9] Loss:  1.38e-03
[Epoch   79/ 100] [Batch    2/   9] Loss:  1.84e-03
[Epoch   79/ 100] [Batch    3/   9] Loss:  1.70e-03
[Epoch   79/ 100] [Batch    4/   9] Loss:  1.57e-03
[Epoch   79/ 100] [Batch    5/   9] Loss:  1.29e-03
[Epoch   79/ 100] [Batch    6/   9] Loss:  1.69e-03
[Epoch   79/ 100] [Batch    7/   9] Loss:  1.68e-03
[Epoch   79/ 100] [Batch    8/   9] Loss:  1.37e-03
[Epoch   79/ 100] [Batch    9/   9] Loss:  1.68e-03
[Epoch   80/ 100] [Batch    1/   9] Loss:  1.43e-03
[Epoch   80/ 100] [Batch    2/   9] Loss:  1.88e-03
[Epoch   80/ 100] [Batch    3/   9] Loss:  1.41e-03
[Epoch   80/ 100] [Batch    4/   9] Loss:  1.70e-03
[Epoch   80/ 100] [Batch    5/   9] Loss:  1.47e-03
[Epoch   80/ 100] [Batch    6/   9] Loss:  1.59e-03
[Epoch   80/ 100] [Batch    7/   9] Loss:  1.38e-03
[Epoch   80/ 100] [Batch    8/   9] Loss:  1.55e-03
[Epoch   80/ 100] [Batch    9/   9] Loss:  1.97e-03
[Epoch   81/ 100] [Batch    1/   9] Loss:  1.54e-03
[Epoch   81/ 100] [Batch    2/   9] Loss:  1.51e-03
[Epoch   81/ 100] [Batch    3/   9] Loss:  1.85e-03
[Epoch   81/ 100] [Batch    4/   9] Loss:  1.40e-03
[Epoch   81/ 100] [Batch    5/   9] Loss:  1.48e-03
[Epoch   81/ 100] [Batch    6/   9] Loss:  1.70e-03
[Epoch   81/ 100] [Batch    7/   9] Loss:  1.53e-03
[Epoch   81/ 100] [Batch    8/   9] Loss:  1.58e-03
[Epoch   81/ 100] [Batch    9/   9] Loss:  1.43e-03
[Epoch   82/ 100] [Batch    1/   9] Loss:  1.58e-03
[Epoch   82/ 100] [Batch    2/   9] Loss:  1.63e-03
[Epoch   82/ 100] [Batch    3/   9] Loss:  1.69e-03
[Epoch   82/ 100] [Batch    4/   9] Loss:  1.56e-03
[Epoch   82/ 100] [Batch    5/   9] Loss:  1.31e-03
[Epoch   82/ 100] [Batch    6/   9] Loss:  1.31e-03
[Epoch   82/ 100] [Batch    7/   9] Loss:  1.32e-03
[Epoch   82/ 100] [Batch    8/   9] Loss:  2.03e-03
[Epoch   82/ 100] [Batch    9/   9] Loss:  1.92e-03
[Epoch   83/ 100] [Batch    1/   9] Loss:  1.30e-03
[Epoch   83/ 100] [Batch    2/   9] Loss:  1.82e-03
[Epoch   83/ 100] [Batch    3/   9] Loss:  1.40e-03
[Epoch   83/ 100] [Batch    4/   9] Loss:  1.65e-03
[Epoch   83/ 100] [Batch    5/   9] Loss:  1.44e-03
[Epoch   83/ 100] [Batch    6/   9] Loss:  1.60e-03
[Epoch   83/ 100] [Batch    7/   9] Loss:  1.62e-03
[Epoch   83/ 100] [Batch    8/   9] Loss:  1.62e-03
[Epoch   83/ 100] [Batch    9/   9] Loss:  2.17e-03
[Epoch   84/ 100] [Batch    1/   9] Loss:  1.19e-03
[Epoch   84/ 100] [Batch    2/   9] Loss:  1.62e-03
[Epoch   84/ 100] [Batch    3/   9] Loss:  1.56e-03
[Epoch   84/ 100] [Batch    4/   9] Loss:  1.78e-03
[Epoch   84/ 100] [Batch    5/   9] Loss:  2.01e-03
[Epoch   84/ 100] [Batch    6/   9] Loss:  1.50e-03
[Epoch   84/ 100] [Batch    7/   9] Loss:  1.23e-03
[Epoch   84/ 100] [Batch    8/   9] Loss:  1.66e-03
[Epoch   84/ 100] [Batch    9/   9] Loss:  1.52e-03
[Epoch   85/ 100] [Batch    1/   9] Loss:  1.60e-03
[Epoch   85/ 100] [Batch    2/   9] Loss:  1.44e-03
[Epoch   85/ 100] [Batch    3/   9] Loss:  1.43e-03
[Epoch   85/ 100] [Batch    4/   9] Loss:  2.02e-03
[Epoch   85/ 100] [Batch    5/   9] Loss:  1.59e-03
[Epoch   85/ 100] [Batch    6/   9] Loss:  1.77e-03
[Epoch   85/ 100] [Batch    7/   9] Loss:  1.41e-03
[Epoch   85/ 100] [Batch    8/   9] Loss:  1.27e-03
[Epoch   85/ 100] [Batch    9/   9] Loss:  1.64e-03
[Epoch   86/ 100] [Batch    1/   9] Loss:  1.39e-03
[Epoch   86/ 100] [Batch    2/   9] Loss:  1.67e-03
[Epoch   86/ 100] [Batch    3/   9] Loss:  1.44e-03
[Epoch   86/ 100] [Batch    4/   9] Loss:  1.77e-03
[Epoch   86/ 100] [Batch    5/   9] Loss:  1.73e-03
[Epoch   86/ 100] [Batch    6/   9] Loss:  1.57e-03
[Epoch   86/ 100] [Batch    7/   9] Loss:  1.59e-03
[Epoch   86/ 100] [Batch    8/   9] Loss:  1.65e-03
[Epoch   86/ 100] [Batch    9/   9] Loss:  8.65e-04
[Epoch   87/ 100] [Batch    1/   9] Loss:  1.65e-03
[Epoch   87/ 100] [Batch    2/   9] Loss:  1.28e-03
[Epoch   87/ 100] [Batch    3/   9] Loss:  1.89e-03
[Epoch   87/ 100] [Batch    4/   9] Loss:  1.28e-03
[Epoch   87/ 100] [Batch    5/   9] Loss:  1.57e-03
[Epoch   87/ 100] [Batch    6/   9] Loss:  1.54e-03
[Epoch   87/ 100] [Batch    7/   9] Loss:  1.57e-03
[Epoch   87/ 100] [Batch    8/   9] Loss:  1.61e-03
[Epoch   87/ 100] [Batch    9/   9] Loss:  2.08e-03
[Epoch   88/ 100] [Batch    1/   9] Loss:  1.64e-03
[Epoch   88/ 100] [Batch    2/   9] Loss:  1.65e-03
[Epoch   88/ 100] [Batch    3/   9] Loss:  1.63e-03
[Epoch   88/ 100] [Batch    4/   9] Loss:  1.76e-03
[Epoch   88/ 100] [Batch    5/   9] Loss:  1.42e-03
[Epoch   88/ 100] [Batch    6/   9] Loss:  1.88e-03
[Epoch   88/ 100] [Batch    7/   9] Loss:  1.33e-03
[Epoch   88/ 100] [Batch    8/   9] Loss:  1.26e-03
[Epoch   88/ 100] [Batch    9/   9] Loss:  1.56e-03
[Epoch   89/ 100] [Batch    1/   9] Loss:  1.69e-03
[Epoch   89/ 100] [Batch    2/   9] Loss:  1.54e-03
[Epoch   89/ 100] [Batch    3/   9] Loss:  1.61e-03
[Epoch   89/ 100] [Batch    4/   9] Loss:  1.61e-03
[Epoch   89/ 100] [Batch    5/   9] Loss:  1.56e-03
[Epoch   89/ 100] [Batch    6/   9] Loss:  1.44e-03
[Epoch   89/ 100] [Batch    7/   9] Loss:  1.38e-03
[Epoch   89/ 100] [Batch    8/   9] Loss:  1.70e-03
[Epoch   89/ 100] [Batch    9/   9] Loss:  1.67e-03
[Epoch   90/ 100] [Batch    1/   9] Loss:  1.62e-03
[Epoch   90/ 100] [Batch    2/   9] Loss:  1.47e-03
[Epoch   90/ 100] [Batch    3/   9] Loss:  1.95e-03
[Epoch   90/ 100] [Batch    4/   9] Loss:  1.46e-03
[Epoch   90/ 100] [Batch    5/   9] Loss:  1.79e-03
[Epoch   90/ 100] [Batch    6/   9] Loss:  1.39e-03
[Epoch   90/ 100] [Batch    7/   9] Loss:  1.67e-03
[Epoch   90/ 100] [Batch    8/   9] Loss:  1.44e-03
[Epoch   90/ 100] [Batch    9/   9] Loss:  8.63e-04
[Epoch   91/ 100] [Batch    1/   9] Loss:  1.48e-03
[Epoch   91/ 100] [Batch    2/   9] Loss:  1.54e-03
[Epoch   91/ 100] [Batch    3/   9] Loss:  1.61e-03
[Epoch   91/ 100] [Batch    4/   9] Loss:  1.37e-03
[Epoch   91/ 100] [Batch    5/   9] Loss:  1.67e-03
[Epoch   91/ 100] [Batch    6/   9] Loss:  1.38e-03
[Epoch   91/ 100] [Batch    7/   9] Loss:  1.87e-03
[Epoch   91/ 100] [Batch    8/   9] Loss:  1.64e-03
[Epoch   91/ 100] [Batch    9/   9] Loss:  1.64e-03
[Epoch   92/ 100] [Batch    1/   9] Loss:  1.44e-03
[Epoch   92/ 100] [Batch    2/   9] Loss:  1.42e-03
[Epoch   92/ 100] [Batch    3/   9] Loss:  1.54e-03
[Epoch   92/ 100] [Batch    4/   9] Loss:  2.00e-03
[Epoch   92/ 100] [Batch    5/   9] Loss:  1.36e-03
[Epoch   92/ 100] [Batch    6/   9] Loss:  1.45e-03
[Epoch   92/ 100] [Batch    7/   9] Loss:  1.65e-03
[Epoch   92/ 100] [Batch    8/   9] Loss:  1.70e-03
[Epoch   92/ 100] [Batch    9/   9] Loss:  2.05e-03
[Epoch   93/ 100] [Batch    1/   9] Loss:  1.47e-03
[Epoch   93/ 100] [Batch    2/   9] Loss:  1.98e-03
[Epoch   93/ 100] [Batch    3/   9] Loss:  1.97e-03
[Epoch   93/ 100] [Batch    4/   9] Loss:  1.72e-03
[Epoch   93/ 100] [Batch    5/   9] Loss:  1.27e-03
[Epoch   93/ 100] [Batch    6/   9] Loss:  1.28e-03
[Epoch   93/ 100] [Batch    7/   9] Loss:  1.35e-03
[Epoch   93/ 100] [Batch    8/   9] Loss:  1.49e-03
[Epoch   93/ 100] [Batch    9/   9] Loss:  1.68e-03
[Epoch   94/ 100] [Batch    1/   9] Loss:  1.58e-03
[Epoch   94/ 100] [Batch    2/   9] Loss:  1.45e-03
[Epoch   94/ 100] [Batch    3/   9] Loss:  1.85e-03
[Epoch   94/ 100] [Batch    4/   9] Loss:  1.91e-03
[Epoch   94/ 100] [Batch    5/   9] Loss:  1.41e-03
[Epoch   94/ 100] [Batch    6/   9] Loss:  1.35e-03
[Epoch   94/ 100] [Batch    7/   9] Loss:  1.32e-03
[Epoch   94/ 100] [Batch    8/   9] Loss:  1.89e-03
[Epoch   94/ 100] [Batch    9/   9] Loss:  9.44e-04
[Epoch   95/ 100] [Batch    1/   9] Loss:  1.38e-03
[Epoch   95/ 100] [Batch    2/   9] Loss:  1.92e-03
[Epoch   95/ 100] [Batch    3/   9] Loss:  1.75e-03
[Epoch   95/ 100] [Batch    4/   9] Loss:  1.66e-03
[Epoch   95/ 100] [Batch    5/   9] Loss:  1.42e-03
[Epoch   95/ 100] [Batch    6/   9] Loss:  1.76e-03
[Epoch   95/ 100] [Batch    7/   9] Loss:  1.22e-03
[Epoch   95/ 100] [Batch    8/   9] Loss:  1.50e-03
[Epoch   95/ 100] [Batch    9/   9] Loss:  1.30e-03
[Epoch   96/ 100] [Batch    1/   9] Loss:  1.37e-03
[Epoch   96/ 100] [Batch    2/   9] Loss:  2.04e-03
[Epoch   96/ 100] [Batch    3/   9] Loss:  1.37e-03
[Epoch   96/ 100] [Batch    4/   9] Loss:  1.51e-03
[Epoch   96/ 100] [Batch    5/   9] Loss:  1.31e-03
[Epoch   96/ 100] [Batch    6/   9] Loss:  1.47e-03
[Epoch   96/ 100] [Batch    7/   9] Loss:  1.55e-03
[Epoch   96/ 100] [Batch    8/   9] Loss:  1.90e-03
[Epoch   96/ 100] [Batch    9/   9] Loss:  1.84e-03
[Epoch   97/ 100] [Batch    1/   9] Loss:  1.33e-03
[Epoch   97/ 100] [Batch    2/   9] Loss:  1.49e-03
[Epoch   97/ 100] [Batch    3/   9] Loss:  1.77e-03
[Epoch   97/ 100] [Batch    4/   9] Loss:  1.69e-03
[Epoch   97/ 100] [Batch    5/   9] Loss:  1.81e-03
[Epoch   97/ 100] [Batch    6/   9] Loss:  1.76e-03
[Epoch   97/ 100] [Batch    7/   9] Loss:  1.24e-03
[Epoch   97/ 100] [Batch    8/   9] Loss:  1.65e-03
[Epoch   97/ 100] [Batch    9/   9] Loss:  1.21e-03
[Epoch   98/ 100] [Batch    1/   9] Loss:  1.61e-03
[Epoch   98/ 100] [Batch    2/   9] Loss:  1.74e-03
[Epoch   98/ 100] [Batch    3/   9] Loss:  1.44e-03
[Epoch   98/ 100] [Batch    4/   9] Loss:  1.61e-03
[Epoch   98/ 100] [Batch    5/   9] Loss:  1.50e-03
[Epoch   98/ 100] [Batch    6/   9] Loss:  1.53e-03
[Epoch   98/ 100] [Batch    7/   9] Loss:  1.48e-03
[Epoch   98/ 100] [Batch    8/   9] Loss:  1.65e-03
[Epoch   98/ 100] [Batch    9/   9] Loss:  1.93e-03
[Epoch   99/ 100] [Batch    1/   9] Loss:  1.62e-03
[Epoch   99/ 100] [Batch    2/   9] Loss:  1.42e-03
[Epoch   99/ 100] [Batch    3/   9] Loss:  1.79e-03
[Epoch   99/ 100] [Batch    4/   9] Loss:  1.71e-03
[Epoch   99/ 100] [Batch    5/   9] Loss:  1.60e-03
[Epoch   99/ 100] [Batch    6/   9] Loss:  1.42e-03
[Epoch   99/ 100] [Batch    7/   9] Loss:  1.39e-03
[Epoch   99/ 100] [Batch    8/   9] Loss:  1.70e-03
[Epoch   99/ 100] [Batch    9/   9] Loss:  1.39e-03
[Epoch  100/ 100] [Batch    1/   9] Loss:  1.66e-03
[Epoch  100/ 100] [Batch    2/   9] Loss:  1.77e-03
[Epoch  100/ 100] [Batch    3/   9] Loss:  1.62e-03
[Epoch  100/ 100] [Batch    4/   9] Loss:  1.37e-03
[Epoch  100/ 100] [Batch    5/   9] Loss:  1.45e-03
[Epoch  100/ 100] [Batch    6/   9] Loss:  1.55e-03
[Epoch  100/ 100] [Batch    7/   9] Loss:  1.56e-03
[Epoch  100/ 100] [Batch    8/   9] Loss:  1.65e-03
[Epoch  100/ 100] [Batch    9/   9] Loss:  1.34e-03
LinearModel(
  (linear): Linear(in_features=1, out_features=1, bias=True)
)
if input_dim == output_dim == 1:
    n_test = 500
    X_test = np.random.rand(n_test, input_dim)
    y_pred = []

    state_dict = model.state_dict()

    for idx in range(len(train_hist['weight'])):
        state_dict['linear.weight'] = torch.tensor([[train_hist['weight'][idx]]])
        state_dict['linear.bias'] = torch.tensor([train_hist['bias'][idx]])
        model.load_state_dict(state_dict)

        y_pred.append(model.forward(torch.tensor(X_test.astype('f'))).detach().numpy())
if input_dim == output_dim == 1:
    fig = plt.figure(figsize=(15, 5))
    fig.clf()

    ax = fig.add_subplot(1, 3, 1)
    levels = np.logspace(np.log(np.min(loss_values)), np.log(np.max(loss_values)), 20)
    ax.contourf(ww, bb, loss_values, levels=levels, norm=colors.LogNorm())
    ax.plot(train_hist['weight'], train_hist['bias'], '.-b')
    ax.plot(A[0], b, 'r*', markersize=10)
    ax.set_xlabel('weight')
    ax.set_ylabel('bias')
    #ax.legend(['optim', '(A, b)'])
    ax.grid(True)
    ax.set_xlim(-2, 2)
    ax.set_ylim(-2, 2)

    ax = fig.add_subplot(1, 3, 2)
    ax.loglog(np.abs(train_hist['loss']))
    ax.set_xlabel('Iter')
    ax.set_ylabel('Loss')
    ax.grid(True)

    ax = fig.add_subplot(1, 3, 3)
    ax.plot(X_train, y_train, '.')
    a=ax.plot(X_test, y_pred[0], '-', alpha=0.1)
    for y in y_pred[1:]:
        ax.plot(X_test, y, '-', alpha=0.1, color=a[0].get_color())
    ax.plot(X_test, y_pred[-1], 'k')
    ax.grid(True)

    fig.tight_layout()

else:
    fig = plt.figure()
    fig.clf()
    ax = fig.gca()
    ax.loglog(np.abs(train_hist['loss']))
    ax.set_xlabel('Iter')
    ax.set_ylabel('Loss')
    ax.grid(True)