PyTorch: Basic examples

Example-1: Scalar uni-variable regression

Import

import torch
import numpy as np
import matplotlib.pyplot as plt

Generate data

# GENERATE DATA TO FIT
x=np.random.uniform(-3,10,100)
p_exact=[1.5,2.5]
y = (p_exact[0]+p_exact[1]*x) + 2.5*np.random.uniform(-1,1,size=len(x))
print("EXACT PARAMETERS:",p_exact)
EXACT PARAMETERS: [1.5, 2.5]

Neural network model

nn.Module is the base class for all neural network modules.

Your models should also subclass this class.

Modules can also contain other Modules, allowing to nest them in a tree structure. You can assign the submodules as regular attributes:

An init() call to the parent class must be made before assignment on the child.

https://pytorch.org/docs/stable/generated/torch.nn.Module.html

# GET CPU OR GPU DEVICE FOR TRAINING.
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"USING {device} DEVICE")

# # Define model
class NeuralNetwork(torch.nn.Module):
    def __init__(self):
        #__init__() call to the parent class
        super(NeuralNetwork, self).__init__()
        #define linear model
        self.linear_model = torch.nn.Sequential(
            torch.nn.Linear(1, 1),
        )

    #FORWARD PASS (EVALUATION OF MODEL)
    def forward(self, x):
        out = self.linear_model(x)
        return out

#INITIALIZE MODEL 
model = NeuralNetwork().to(device)
print(model)
USING cpu DEVICE
NeuralNetwork(
  (linear_model): Sequential(
    (0): Linear(in_features=1, out_features=1, bias=True)
  )
)
#CHECK PARAMETERS (NOTICE HOW THEY WERE AUTOMATICALLY RANDOMIZED)
for param in model.parameters():
    print(param)
    print(param.shape)
    print(param.requires_grad)
Parameter containing:
tensor([[-0.4367]], requires_grad=True)
torch.Size([1, 1])
True
Parameter containing:
tensor([-0.3906], requires_grad=True)
torch.Size([1])
True
# CONVERT TO TORCH TENSOR AND EVALUATE MODEL AS A TEST
x=torch.tensor(x).view(x.shape[0],1)
y=torch.tensor(y).view(x.shape[0],1)

from sklearn.model_selection import train_test_split
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=0)

print("X-VALIDATION:",x_val.shape,x_val.dtype)
print("Y-VALIDATION:",y_val.shape,y_val.dtype)
print("X-TRAIN:",x_train.shape,x_train.dtype)
print("Y-TRAIN:",y_train.shape,y_train.dtype)
y_pred = model(x_train.float())
print("PREDICTION:",y_pred.shape,y_pred.dtype)
X-VALIDATION: torch.Size([20, 1]) torch.float64
Y-VALIDATION: torch.Size([20, 1]) torch.float64
X-TRAIN: torch.Size([80, 1]) torch.float64
Y-TRAIN: torch.Size([80, 1]) torch.float64
PREDICTION: torch.Size([80, 1]) torch.float32
# PLOT THE FIGURE
fig, ax = plt.subplots()
ax.plot(x_train, y_train, 'o', label='training Data')
ax.plot(x_val, y_val, 'o', label='validation Data')
ax.plot(x_train, y_pred.detach().numpy(), '-', label='predictions for initial parameters')
ax.legend()
plt.show()

Initialize


#INITIALIZE ARRAYS TO SAVE HISTORY
epoch_array=[]
val_loss_history=[]
train_loss_history=[]

#INITIALIZE LOSS FUNCTION AND OPTIMIZER
loss_fn = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)

Utility functions

i_print=True

def train(x_train,y_train, model, loss_fn, optimizer):
    #TELL MODEL IT SHOULD BE TRAINING
    model.train()

    #BATCH IMPLEMENTATION 
    X, y = x_train.to(device).float(), y_train.to(device).float()

    # Compute prediction error
    pred = model(X)
    loss = loss_fn(pred, y)

    # Backpropagation
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    # if t % 5 == 0:
    loss = loss.item() 
    train_loss_history.append(loss)

    if(i_print):
        print(f"loss: {loss:>7f}")
def validate(x_val,y_val, model, loss_fn):
    #TELL MODEL IT SHOULD BE EVALUATING
    model.eval()
    X, y = x_val.to(device).float(), y_val.to(device).float()
    pred = model(X)
    val_loss = loss_fn(pred, y).item()
    val_loss_history.append(val_loss)
    if(i_print):
        print(f"Test Error: {val_loss:>8f}")

    #FOR CLASSIFICATION 
    # correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    # correct /= y.shape[0]
    # print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

Training loop

epochs = 300

for t in range(epochs):
    epoch_array.append(t)
    if t%50==0:
        i_print=True
        print(f"-------------------------------\nEpoch {t}")

    else: 
        i_print=False

    train(x_train,y_train, model, loss_fn, optimizer)
    validate(x_val,y_val, model, loss_fn)
print("Done!")
-------------------------------
Epoch 0
loss: 234.110596
Test Error: 229.450775
-------------------------------
Epoch 50
loss: 5.418958
Test Error: 5.860951
-------------------------------
Epoch 100
loss: 3.070067
Test Error: 3.039767
-------------------------------
Epoch 150
loss: 2.957608
Test Error: 2.881173
-------------------------------
Epoch 200
loss: 2.875188
Test Error: 2.805054
-------------------------------
Epoch 250
loss: 2.800372
Test Error: 2.741704
Done!

Plot results

# PLOT THE FIGURE
y_pred = model(x_train.float())

# PLOT THE FIGURE
fig, ax = plt.subplots()
ax.plot(x_train, y_train, 'o', label='training Data')
ax.plot(x_val, y_val, 'o', label='validation Data')
ax.plot(x_train, y_pred.detach().numpy(), '-', label='predictions')
ax.legend()
plt.show()

# PLOT THE FIGURE
y_pred = model(x_train.float())

fig, ax = plt.subplots()
ax.plot(epoch_array, train_loss_history, '-', label='training loss')
ax.plot(epoch_array, val_loss_history, '-', label='validation loss')
ax.legend()
plt.show()

Pytorch Scalar multi-variable non-linear regression

Import

import torch
import numpy as np
import matplotlib.pyplot as plt

Generate data

# GENERATE DATA TO FIT
x=np.random.uniform(-10,10,(100,2))
print(x.shape)
pe=np.random.uniform(-3,3,(6,))
y = pe[0]*x[:,0]**2 + pe[1]*x[:,0]*x[:,1] +pe[2]*x[:,1]**2 + pe[3]*x[:,0] + pe[4]*x[:,1] + pe[5] 
y += 0.04*np.random.uniform(np.min(y),np.max(y),size=len(x))
print("EXACT PARAMETERS:",pe)
print(x)
(100, 2)
EXACT PARAMETERS: [ 1.48424549 -0.52642468  1.58383604 -2.3825064  -1.10378184 -0.68543161]
[[ 8.17378877 -5.45853478]
 [-4.59060502  7.66478271]
 [ 3.25365845 -3.21351797]
 [ 8.08717117  9.11762947]
 [-3.65942736 -6.81428213]
 [ 5.60488818  6.63082577]
 [-8.04942154 -8.47195008]
 [-4.06264164  1.46180593]
 [ 9.87222627  3.69222276]
 [-4.78284394 -1.90554577]
 [-6.04665051 -1.11524591]
 [-4.14241574 -3.23007605]
 [ 2.23680302 -7.49154583]
 [-0.06293325 -8.15410595]
 [-7.64724124 -0.42344715]
 [-1.94249079  8.05638525]
 [ 1.26767217  9.7469471 ]
 [-9.09385814 -6.51744688]
 [-4.5643795  -8.50473635]
 [ 2.01718664  2.58830187]
 [-6.32928313 -3.42331548]
 [-7.99793927 -2.4239683 ]
 [ 3.8320992   3.24578486]
 [ 4.05302967  5.02704801]
 [-7.83810849  6.0126653 ]
 [ 5.71957109  1.74423207]
 [-1.15592833 -7.70285235]
 [-5.95403716  4.95223725]
 [-9.90352937 -0.46157685]
 [-9.16373035  4.05902768]
 [ 5.7591351  -0.34836824]
 [-8.83399069  0.50648637]
 [-4.83943226 -4.97279239]
 [ 2.98766038  8.2496762 ]
 [-5.35444439  1.01066082]
 [ 1.02297882 -9.49674089]
 [-4.460375    8.8559558 ]
 [-3.06894132 -6.67171379]
 [ 2.72716413 -9.12098931]
 [ 1.96245866  0.0631399 ]
 [ 8.36361022  4.76903293]
 [ 6.54747238  9.81194098]
 [ 0.56721624 -1.37164318]
 [ 3.33325725 -2.51847818]
 [ 2.48492764 -3.37480525]
 [-0.95926276 -4.66667934]
 [-1.43376244  8.63831965]
 [-4.45940715  2.22130783]
 [ 7.9838345  -0.86714468]
 [-7.84086586 -5.1273224 ]
 [-4.9370314   3.56646294]
 [-3.95325696  9.09157567]
 [ 7.19919078  0.12829911]
 [-7.24116608  6.6049262 ]
 [-3.00998674  6.73799407]
 [ 4.89807643  8.10694516]
 [ 7.2993556  -9.82585387]
 [ 5.28602554  3.37857916]
 [ 8.47211933 -8.26707666]
 [-7.98390606 -9.02402781]
 [ 0.74372697  2.86064999]
 [ 9.57429236 -2.93044888]
 [-8.10251825 -1.13721307]
 [-3.93154564 -9.92950247]
 [-3.06147617 -3.49156469]
 [-9.98750407 -6.44129652]
 [-3.84718663 -6.14536021]
 [ 9.64933147 -1.49025683]
 [ 6.08315503  6.85308411]
 [-1.62101442  1.23858672]
 [ 4.01746231  5.34749594]
 [ 7.91396954 -8.57141268]
 [ 9.86077176 -8.52365132]
 [ 1.96594972 -1.81590597]
 [-5.72493147  6.00120459]
 [-1.02510876 -3.65910306]
 [-1.88158704  2.33640902]
 [-4.18971035 -0.17663683]
 [-6.06194325  0.55922549]
 [ 5.02600052  5.52518953]
 [-1.64289366 -4.76889639]
 [-5.9820669  -1.78682783]
 [-4.46844811  4.15246825]
 [-2.16165205 -8.6896733 ]
 [-7.9499032  -3.21225848]
 [ 4.46173415 -2.1063864 ]
 [ 8.14351935  4.06823292]
 [-4.2153306  -1.02256414]
 [-5.51019978  0.89451637]
 [-2.56035793 -9.52225953]
 [-9.36494831 -9.96719938]
 [-5.1372779   7.32216767]
 [-8.17180059 -8.65626827]
 [ 3.13345046  2.57674889]
 [ 0.96839083  6.99607017]
 [ 4.62142677 -1.45398315]
 [-6.02022462  6.97580991]
 [-0.92293929  7.10385766]
 [-4.94391004  5.28791512]
 [ 4.58833367 -5.80125927]]

Neural network model

nn.Module is the base class for all neural network modules.

Your models should also subclass this class.

Modules can also contain other Modules, allowing to nest them in a tree structure. You can assign the submodules as regular attributes:

An init() call to the parent class must be made before assignment on the child.

https://pytorch.org/docs/stable/generated/torch.nn.Module.html

# GET CPU OR GPU DEVICE FOR TRAINING.
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"USING {device} DEVICE")

input_dimension=x.shape[1]
print(input_dimension)
USING cpu DEVICE
2


# # Define model
class NeuralNetwork(torch.nn.Module):
    def __init__(self):
        #__init__() call to the parent class
        super(NeuralNetwork, self).__init__()
        #define linear model
        self.linear_model = torch.nn.Sequential(
            torch.nn.Linear(input_dimension, 6),
            torch.nn.Sigmoid(),
            torch.nn.Linear(6, 6),
            torch.nn.Sigmoid(),
            torch.nn.Linear(6, 1),
        )

    #FORWARD PASS (EVALUATION OF MODEL)
    def forward(self, x):
        out = self.linear_model(x)
        return out

#INITIALIZE MODEL 
model = NeuralNetwork().to(device)
print(model)
NeuralNetwork(
  (linear_model): Sequential(
    (0): Linear(in_features=2, out_features=6, bias=True)
    (1): Sigmoid()
    (2): Linear(in_features=6, out_features=6, bias=True)
    (3): Sigmoid()
    (4): Linear(in_features=6, out_features=1, bias=True)
  )
)
#CHECK PARAMETERS (NOTICE HOW THEY WERE AUTOMATICALLY RANDOMIZED)
for param in model.parameters():
    print(param)
    print(param.shape)
    print(param.requires_grad)
Parameter containing:
tensor([[ 0.4541,  0.5148],
        [ 0.4872,  0.5106],
        [ 0.3607,  0.1465],
        [ 0.0398, -0.1825],
        [ 0.1908, -0.6027],
        [-0.0740,  0.4415]], requires_grad=True)
torch.Size([6, 2])
True
Parameter containing:
tensor([-0.2243, -0.3142, -0.6364,  0.2573,  0.5157, -0.0563],
       requires_grad=True)
torch.Size([6])
True
Parameter containing:
tensor([[ 0.3289,  0.2376,  0.0888,  0.3899, -0.3680,  0.0681],
        [ 0.1339, -0.3545,  0.2536,  0.0281, -0.0449,  0.0652],
        [-0.0132, -0.1867,  0.3049,  0.1733,  0.0023,  0.2861],
        [-0.1059,  0.0495,  0.0214,  0.3953, -0.2238, -0.0859],
        [-0.3150, -0.1545, -0.3688,  0.3879,  0.3661, -0.3421],
        [ 0.0740, -0.3250, -0.3680,  0.3849, -0.0776,  0.1029]],
       requires_grad=True)
torch.Size([6, 6])
True
Parameter containing:
tensor([-0.2774,  0.1523,  0.2906, -0.3929,  0.1072, -0.3869],
       requires_grad=True)
torch.Size([6])
True
Parameter containing:
tensor([[ 0.3580, -0.3169, -0.3630, -0.0648, -0.1032, -0.3982]],
       requires_grad=True)
torch.Size([1, 6])
True
Parameter containing:
tensor([0.4015], requires_grad=True)
torch.Size([1])
True
# CONVERT TO TORCH TENSOR AND EVALUATE MODEL AS A TEST
x=torch.tensor(x).view(x.shape[0],x.shape[1])
y=torch.tensor(y).view(x.shape[0],1)

from sklearn.model_selection import train_test_split
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=0)

print("X-VALIDATION:",x_val.shape,x_val.dtype)
print("Y-VALIDATION:",y_val.shape,y_val.dtype)
print("X-TRAIN:",x_train.shape,x_train.dtype)
print("Y-TRAIN:",y_train.shape,y_train.dtype)
y_pred = model(x_train.float())
print("PREDICTION:",y_pred.shape,y_pred.dtype)
X-VALIDATION: torch.Size([20, 2]) torch.float64
Y-VALIDATION: torch.Size([20, 1]) torch.float64
X-TRAIN: torch.Size([80, 2]) torch.float64
Y-TRAIN: torch.Size([80, 1]) torch.float64
PREDICTION: torch.Size([80, 1]) torch.float32
# PLOT THE FIGURE
# ax.scatter(x_val[:,0], x_val[:,1], y_val, c = 'r', marker='o')
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x_train[:,0], x_train[:,1], y_train, c = 'b', marker='o')
ax.scatter(x_train[:,0], x_train[:,1], y_pred.detach().numpy(), c = 'r', marker='o')
plt.show()

Initialize


#INITIALIZE ARRAYS TO SAVE HISTORY
epoch_array=[]
val_loss_history=[]
train_loss_history=[]

#INITIALIZE LOSS FUNCTION AND OPTIMIZER
loss_fn = torch.nn.MSELoss()
optimizer = torch.optim.RMSprop(model.parameters(), lr=1e-2)

Utility functions

i_print=True

def train(x_train,y_train, model, loss_fn, optimizer):
    #TELL MODEL IT SHOULD BE TRAINING
    model.train()

    #BATCH IMPLEMENTATION 
    X, y = x_train.to(device).float(), y_train.to(device).float()

    # Compute prediction error
    pred = model(X)
    loss = loss_fn(pred, y)

    # Backpropagation
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    # if t % 5 == 0:
    loss = loss.item() 
    train_loss_history.append(loss)

    if(i_print):
        print(f"loss: {loss:>7f}")
def validate(x_val,y_val, model, loss_fn):
    #TELL MODEL IT SHOULD BE EVALUATING
    model.eval()
    X, y = x_val.to(device).float(), y_val.to(device).float()
    pred = model(X)
    val_loss = loss_fn(pred, y).item()
    val_loss_history.append(val_loss)
    if(i_print):
        print(f"Test Error: {val_loss:>8f}")

    #FOR CLASSIFICATION 
    # correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    # correct /= y.shape[0]
    # print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

Training loop

epochs = 6000

for t in range(epochs):
    epoch_array.append(t)
    if t%500==0:
        i_print=True
        print(f"-------------------------------\nEpoch {t}")

    else: 
        i_print=False

    train(x_train,y_train, model, loss_fn, optimizer)
    validate(x_val,y_val, model, loss_fn)
print("Done!")
-------------------------------
Epoch 0
loss: 17286.628906
Test Error: 11484.698242
-------------------------------
Epoch 500
loss: 9467.769531
Test Error: 5751.200195
-------------------------------
Epoch 1000
loss: 6083.802246
Test Error: 3825.845703
-------------------------------
Epoch 1500
loss: 4714.020020
Test Error: 3737.626465
-------------------------------
Epoch 2000
loss: 4307.856445
Test Error: 3794.660156
-------------------------------
Epoch 2500
loss: 1088.462524
Test Error: 417.059326
-------------------------------
Epoch 3000
loss: 549.802124
Test Error: 124.163292
-------------------------------
Epoch 3500
loss: 279.107910
Test Error: 86.994858
-------------------------------
Epoch 4000
loss: 159.840408
Test Error: 116.015503
-------------------------------
Epoch 4500
loss: 88.524094
Test Error: 118.878708
-------------------------------
Epoch 5000
loss: 62.226372
Test Error: 85.358681
-------------------------------
Epoch 5500
loss: 58.749115
Test Error: 72.058266
Done!

Plot results

# PLOT THE FIGURE
y_pred = model(x_train.float())

# PLOT THE FIGURE
# ax.scatter(x_val[:,0], x_val[:,1], y_val, c = 'r', marker='o')
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x_train[:,0], x_train[:,1], y_train, c = 'b', marker='o')
ax.scatter(x_train[:,0], x_train[:,1], y_pred.detach().numpy(), c = 'r', marker='o')
plt.show()

# PLOT THE FIGURE
y_pred = model(x_train.float())

fig, ax = plt.subplots()
ax.plot(epoch_array, train_loss_history, '.', label='training loss')
ax.plot(epoch_array, val_loss_history, '.', label='validation loss')
ax.legend()
plt.show()