import torch
import numpy as np
import matplotlib.pyplot as pltPyTorch: Basic examples
- Author: J. Hickman
- scalar linear regression m(x,p)=p[0]+p[1]*x
Example-1: Scalar uni-variable regression
Import
Generate data
# GENERATE DATA TO FIT
x=np.random.uniform(-3,10,100)
p_exact=[1.5,2.5]
y = (p_exact[0]+p_exact[1]*x) + 2.5*np.random.uniform(-1,1,size=len(x))
print("EXACT PARAMETERS:",p_exact)EXACT PARAMETERS: [1.5, 2.5]
Neural network model
nn.Module is the base class for all neural network modules.
Your models should also subclass this class.
Modules can also contain other Modules, allowing to nest them in a tree structure. You can assign the submodules as regular attributes:
An init() call to the parent class must be made before assignment on the child.
https://pytorch.org/docs/stable/generated/torch.nn.Module.html
# GET CPU OR GPU DEVICE FOR TRAINING.
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"USING {device} DEVICE")
# # Define model
class NeuralNetwork(torch.nn.Module):
def __init__(self):
#__init__() call to the parent class
super(NeuralNetwork, self).__init__()
#define linear model
self.linear_model = torch.nn.Sequential(
torch.nn.Linear(1, 1),
)
#FORWARD PASS (EVALUATION OF MODEL)
def forward(self, x):
out = self.linear_model(x)
return out
#INITIALIZE MODEL
model = NeuralNetwork().to(device)
print(model)USING cpu DEVICE
NeuralNetwork(
(linear_model): Sequential(
(0): Linear(in_features=1, out_features=1, bias=True)
)
)
#CHECK PARAMETERS (NOTICE HOW THEY WERE AUTOMATICALLY RANDOMIZED)
for param in model.parameters():
print(param)
print(param.shape)
print(param.requires_grad)Parameter containing:
tensor([[-0.4367]], requires_grad=True)
torch.Size([1, 1])
True
Parameter containing:
tensor([-0.3906], requires_grad=True)
torch.Size([1])
True
# CONVERT TO TORCH TENSOR AND EVALUATE MODEL AS A TEST
x=torch.tensor(x).view(x.shape[0],1)
y=torch.tensor(y).view(x.shape[0],1)
from sklearn.model_selection import train_test_split
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=0)
print("X-VALIDATION:",x_val.shape,x_val.dtype)
print("Y-VALIDATION:",y_val.shape,y_val.dtype)
print("X-TRAIN:",x_train.shape,x_train.dtype)
print("Y-TRAIN:",y_train.shape,y_train.dtype)
y_pred = model(x_train.float())
print("PREDICTION:",y_pred.shape,y_pred.dtype)X-VALIDATION: torch.Size([20, 1]) torch.float64
Y-VALIDATION: torch.Size([20, 1]) torch.float64
X-TRAIN: torch.Size([80, 1]) torch.float64
Y-TRAIN: torch.Size([80, 1]) torch.float64
PREDICTION: torch.Size([80, 1]) torch.float32
# PLOT THE FIGURE
fig, ax = plt.subplots()
ax.plot(x_train, y_train, 'o', label='training Data')
ax.plot(x_val, y_val, 'o', label='validation Data')
ax.plot(x_train, y_pred.detach().numpy(), '-', label='predictions for initial parameters')
ax.legend()
plt.show()
Initialize
#INITIALIZE ARRAYS TO SAVE HISTORY
epoch_array=[]
val_loss_history=[]
train_loss_history=[]
#INITIALIZE LOSS FUNCTION AND OPTIMIZER
loss_fn = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)Utility functions
i_print=True
def train(x_train,y_train, model, loss_fn, optimizer):
#TELL MODEL IT SHOULD BE TRAINING
model.train()
#BATCH IMPLEMENTATION
X, y = x_train.to(device).float(), y_train.to(device).float()
# Compute prediction error
pred = model(X)
loss = loss_fn(pred, y)
# Backpropagation
optimizer.zero_grad()
loss.backward()
optimizer.step()
# if t % 5 == 0:
loss = loss.item()
train_loss_history.append(loss)
if(i_print):
print(f"loss: {loss:>7f}")def validate(x_val,y_val, model, loss_fn):
#TELL MODEL IT SHOULD BE EVALUATING
model.eval()
X, y = x_val.to(device).float(), y_val.to(device).float()
pred = model(X)
val_loss = loss_fn(pred, y).item()
val_loss_history.append(val_loss)
if(i_print):
print(f"Test Error: {val_loss:>8f}")
#FOR CLASSIFICATION
# correct += (pred.argmax(1) == y).type(torch.float).sum().item()
# correct /= y.shape[0]
# print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")Training loop
epochs = 300
for t in range(epochs):
epoch_array.append(t)
if t%50==0:
i_print=True
print(f"-------------------------------\nEpoch {t}")
else:
i_print=False
train(x_train,y_train, model, loss_fn, optimizer)
validate(x_val,y_val, model, loss_fn)
print("Done!")-------------------------------
Epoch 0
loss: 234.110596
Test Error: 229.450775
-------------------------------
Epoch 50
loss: 5.418958
Test Error: 5.860951
-------------------------------
Epoch 100
loss: 3.070067
Test Error: 3.039767
-------------------------------
Epoch 150
loss: 2.957608
Test Error: 2.881173
-------------------------------
Epoch 200
loss: 2.875188
Test Error: 2.805054
-------------------------------
Epoch 250
loss: 2.800372
Test Error: 2.741704
Done!
Plot results
# PLOT THE FIGURE
y_pred = model(x_train.float())
# PLOT THE FIGURE
fig, ax = plt.subplots()
ax.plot(x_train, y_train, 'o', label='training Data')
ax.plot(x_val, y_val, 'o', label='validation Data')
ax.plot(x_train, y_pred.detach().numpy(), '-', label='predictions')
ax.legend()
plt.show()
# PLOT THE FIGURE
y_pred = model(x_train.float())
fig, ax = plt.subplots()
ax.plot(epoch_array, train_loss_history, '-', label='training loss')
ax.plot(epoch_array, val_loss_history, '-', label='validation loss')
ax.legend()
plt.show()
Pytorch Scalar multi-variable non-linear regression
Import
import torch
import numpy as np
import matplotlib.pyplot as pltGenerate data
# GENERATE DATA TO FIT
x=np.random.uniform(-10,10,(100,2))
print(x.shape)
pe=np.random.uniform(-3,3,(6,))
y = pe[0]*x[:,0]**2 + pe[1]*x[:,0]*x[:,1] +pe[2]*x[:,1]**2 + pe[3]*x[:,0] + pe[4]*x[:,1] + pe[5]
y += 0.04*np.random.uniform(np.min(y),np.max(y),size=len(x))
print("EXACT PARAMETERS:",pe)
print(x)(100, 2)
EXACT PARAMETERS: [ 1.48424549 -0.52642468 1.58383604 -2.3825064 -1.10378184 -0.68543161]
[[ 8.17378877 -5.45853478]
[-4.59060502 7.66478271]
[ 3.25365845 -3.21351797]
[ 8.08717117 9.11762947]
[-3.65942736 -6.81428213]
[ 5.60488818 6.63082577]
[-8.04942154 -8.47195008]
[-4.06264164 1.46180593]
[ 9.87222627 3.69222276]
[-4.78284394 -1.90554577]
[-6.04665051 -1.11524591]
[-4.14241574 -3.23007605]
[ 2.23680302 -7.49154583]
[-0.06293325 -8.15410595]
[-7.64724124 -0.42344715]
[-1.94249079 8.05638525]
[ 1.26767217 9.7469471 ]
[-9.09385814 -6.51744688]
[-4.5643795 -8.50473635]
[ 2.01718664 2.58830187]
[-6.32928313 -3.42331548]
[-7.99793927 -2.4239683 ]
[ 3.8320992 3.24578486]
[ 4.05302967 5.02704801]
[-7.83810849 6.0126653 ]
[ 5.71957109 1.74423207]
[-1.15592833 -7.70285235]
[-5.95403716 4.95223725]
[-9.90352937 -0.46157685]
[-9.16373035 4.05902768]
[ 5.7591351 -0.34836824]
[-8.83399069 0.50648637]
[-4.83943226 -4.97279239]
[ 2.98766038 8.2496762 ]
[-5.35444439 1.01066082]
[ 1.02297882 -9.49674089]
[-4.460375 8.8559558 ]
[-3.06894132 -6.67171379]
[ 2.72716413 -9.12098931]
[ 1.96245866 0.0631399 ]
[ 8.36361022 4.76903293]
[ 6.54747238 9.81194098]
[ 0.56721624 -1.37164318]
[ 3.33325725 -2.51847818]
[ 2.48492764 -3.37480525]
[-0.95926276 -4.66667934]
[-1.43376244 8.63831965]
[-4.45940715 2.22130783]
[ 7.9838345 -0.86714468]
[-7.84086586 -5.1273224 ]
[-4.9370314 3.56646294]
[-3.95325696 9.09157567]
[ 7.19919078 0.12829911]
[-7.24116608 6.6049262 ]
[-3.00998674 6.73799407]
[ 4.89807643 8.10694516]
[ 7.2993556 -9.82585387]
[ 5.28602554 3.37857916]
[ 8.47211933 -8.26707666]
[-7.98390606 -9.02402781]
[ 0.74372697 2.86064999]
[ 9.57429236 -2.93044888]
[-8.10251825 -1.13721307]
[-3.93154564 -9.92950247]
[-3.06147617 -3.49156469]
[-9.98750407 -6.44129652]
[-3.84718663 -6.14536021]
[ 9.64933147 -1.49025683]
[ 6.08315503 6.85308411]
[-1.62101442 1.23858672]
[ 4.01746231 5.34749594]
[ 7.91396954 -8.57141268]
[ 9.86077176 -8.52365132]
[ 1.96594972 -1.81590597]
[-5.72493147 6.00120459]
[-1.02510876 -3.65910306]
[-1.88158704 2.33640902]
[-4.18971035 -0.17663683]
[-6.06194325 0.55922549]
[ 5.02600052 5.52518953]
[-1.64289366 -4.76889639]
[-5.9820669 -1.78682783]
[-4.46844811 4.15246825]
[-2.16165205 -8.6896733 ]
[-7.9499032 -3.21225848]
[ 4.46173415 -2.1063864 ]
[ 8.14351935 4.06823292]
[-4.2153306 -1.02256414]
[-5.51019978 0.89451637]
[-2.56035793 -9.52225953]
[-9.36494831 -9.96719938]
[-5.1372779 7.32216767]
[-8.17180059 -8.65626827]
[ 3.13345046 2.57674889]
[ 0.96839083 6.99607017]
[ 4.62142677 -1.45398315]
[-6.02022462 6.97580991]
[-0.92293929 7.10385766]
[-4.94391004 5.28791512]
[ 4.58833367 -5.80125927]]
Neural network model
nn.Module is the base class for all neural network modules.
Your models should also subclass this class.
Modules can also contain other Modules, allowing to nest them in a tree structure. You can assign the submodules as regular attributes:
An init() call to the parent class must be made before assignment on the child.
https://pytorch.org/docs/stable/generated/torch.nn.Module.html
# GET CPU OR GPU DEVICE FOR TRAINING.
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"USING {device} DEVICE")
input_dimension=x.shape[1]
print(input_dimension)USING cpu DEVICE
2
# # Define model
class NeuralNetwork(torch.nn.Module):
def __init__(self):
#__init__() call to the parent class
super(NeuralNetwork, self).__init__()
#define linear model
self.linear_model = torch.nn.Sequential(
torch.nn.Linear(input_dimension, 6),
torch.nn.Sigmoid(),
torch.nn.Linear(6, 6),
torch.nn.Sigmoid(),
torch.nn.Linear(6, 1),
)
#FORWARD PASS (EVALUATION OF MODEL)
def forward(self, x):
out = self.linear_model(x)
return out
#INITIALIZE MODEL
model = NeuralNetwork().to(device)
print(model)NeuralNetwork(
(linear_model): Sequential(
(0): Linear(in_features=2, out_features=6, bias=True)
(1): Sigmoid()
(2): Linear(in_features=6, out_features=6, bias=True)
(3): Sigmoid()
(4): Linear(in_features=6, out_features=1, bias=True)
)
)
#CHECK PARAMETERS (NOTICE HOW THEY WERE AUTOMATICALLY RANDOMIZED)
for param in model.parameters():
print(param)
print(param.shape)
print(param.requires_grad)Parameter containing:
tensor([[ 0.4541, 0.5148],
[ 0.4872, 0.5106],
[ 0.3607, 0.1465],
[ 0.0398, -0.1825],
[ 0.1908, -0.6027],
[-0.0740, 0.4415]], requires_grad=True)
torch.Size([6, 2])
True
Parameter containing:
tensor([-0.2243, -0.3142, -0.6364, 0.2573, 0.5157, -0.0563],
requires_grad=True)
torch.Size([6])
True
Parameter containing:
tensor([[ 0.3289, 0.2376, 0.0888, 0.3899, -0.3680, 0.0681],
[ 0.1339, -0.3545, 0.2536, 0.0281, -0.0449, 0.0652],
[-0.0132, -0.1867, 0.3049, 0.1733, 0.0023, 0.2861],
[-0.1059, 0.0495, 0.0214, 0.3953, -0.2238, -0.0859],
[-0.3150, -0.1545, -0.3688, 0.3879, 0.3661, -0.3421],
[ 0.0740, -0.3250, -0.3680, 0.3849, -0.0776, 0.1029]],
requires_grad=True)
torch.Size([6, 6])
True
Parameter containing:
tensor([-0.2774, 0.1523, 0.2906, -0.3929, 0.1072, -0.3869],
requires_grad=True)
torch.Size([6])
True
Parameter containing:
tensor([[ 0.3580, -0.3169, -0.3630, -0.0648, -0.1032, -0.3982]],
requires_grad=True)
torch.Size([1, 6])
True
Parameter containing:
tensor([0.4015], requires_grad=True)
torch.Size([1])
True
# CONVERT TO TORCH TENSOR AND EVALUATE MODEL AS A TEST
x=torch.tensor(x).view(x.shape[0],x.shape[1])
y=torch.tensor(y).view(x.shape[0],1)
from sklearn.model_selection import train_test_split
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=0)
print("X-VALIDATION:",x_val.shape,x_val.dtype)
print("Y-VALIDATION:",y_val.shape,y_val.dtype)
print("X-TRAIN:",x_train.shape,x_train.dtype)
print("Y-TRAIN:",y_train.shape,y_train.dtype)
y_pred = model(x_train.float())
print("PREDICTION:",y_pred.shape,y_pred.dtype)X-VALIDATION: torch.Size([20, 2]) torch.float64
Y-VALIDATION: torch.Size([20, 1]) torch.float64
X-TRAIN: torch.Size([80, 2]) torch.float64
Y-TRAIN: torch.Size([80, 1]) torch.float64
PREDICTION: torch.Size([80, 1]) torch.float32
# PLOT THE FIGURE
# ax.scatter(x_val[:,0], x_val[:,1], y_val, c = 'r', marker='o')
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x_train[:,0], x_train[:,1], y_train, c = 'b', marker='o')
ax.scatter(x_train[:,0], x_train[:,1], y_pred.detach().numpy(), c = 'r', marker='o')
plt.show()
Initialize
#INITIALIZE ARRAYS TO SAVE HISTORY
epoch_array=[]
val_loss_history=[]
train_loss_history=[]
#INITIALIZE LOSS FUNCTION AND OPTIMIZER
loss_fn = torch.nn.MSELoss()
optimizer = torch.optim.RMSprop(model.parameters(), lr=1e-2)Utility functions
i_print=True
def train(x_train,y_train, model, loss_fn, optimizer):
#TELL MODEL IT SHOULD BE TRAINING
model.train()
#BATCH IMPLEMENTATION
X, y = x_train.to(device).float(), y_train.to(device).float()
# Compute prediction error
pred = model(X)
loss = loss_fn(pred, y)
# Backpropagation
optimizer.zero_grad()
loss.backward()
optimizer.step()
# if t % 5 == 0:
loss = loss.item()
train_loss_history.append(loss)
if(i_print):
print(f"loss: {loss:>7f}")def validate(x_val,y_val, model, loss_fn):
#TELL MODEL IT SHOULD BE EVALUATING
model.eval()
X, y = x_val.to(device).float(), y_val.to(device).float()
pred = model(X)
val_loss = loss_fn(pred, y).item()
val_loss_history.append(val_loss)
if(i_print):
print(f"Test Error: {val_loss:>8f}")
#FOR CLASSIFICATION
# correct += (pred.argmax(1) == y).type(torch.float).sum().item()
# correct /= y.shape[0]
# print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")Training loop
epochs = 6000
for t in range(epochs):
epoch_array.append(t)
if t%500==0:
i_print=True
print(f"-------------------------------\nEpoch {t}")
else:
i_print=False
train(x_train,y_train, model, loss_fn, optimizer)
validate(x_val,y_val, model, loss_fn)
print("Done!")-------------------------------
Epoch 0
loss: 17286.628906
Test Error: 11484.698242
-------------------------------
Epoch 500
loss: 9467.769531
Test Error: 5751.200195
-------------------------------
Epoch 1000
loss: 6083.802246
Test Error: 3825.845703
-------------------------------
Epoch 1500
loss: 4714.020020
Test Error: 3737.626465
-------------------------------
Epoch 2000
loss: 4307.856445
Test Error: 3794.660156
-------------------------------
Epoch 2500
loss: 1088.462524
Test Error: 417.059326
-------------------------------
Epoch 3000
loss: 549.802124
Test Error: 124.163292
-------------------------------
Epoch 3500
loss: 279.107910
Test Error: 86.994858
-------------------------------
Epoch 4000
loss: 159.840408
Test Error: 116.015503
-------------------------------
Epoch 4500
loss: 88.524094
Test Error: 118.878708
-------------------------------
Epoch 5000
loss: 62.226372
Test Error: 85.358681
-------------------------------
Epoch 5500
loss: 58.749115
Test Error: 72.058266
Done!
Plot results
# PLOT THE FIGURE
y_pred = model(x_train.float())
# PLOT THE FIGURE
# ax.scatter(x_val[:,0], x_val[:,1], y_val, c = 'r', marker='o')
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x_train[:,0], x_train[:,1], y_train, c = 'b', marker='o')
ax.scatter(x_train[:,0], x_train[:,1], y_pred.detach().numpy(), c = 'r', marker='o')
plt.show()
# PLOT THE FIGURE
y_pred = model(x_train.float())
fig, ax = plt.subplots()
ax.plot(epoch_array, train_loss_history, '.', label='training loss')
ax.plot(epoch_array, val_loss_history, '.', label='validation loss')
ax.legend()
plt.show()