← На главную Воспоминания

Воскресенье, 9 Июня 2024

Прошел нулевую часть курса по пайторчу. Вот ноутбук

import torch
torch.__version__
'2.1.2+cpu'

Tensors

# Scalar
scalar = torch.tensor(7)
print(scalar)
print(scalar.ndim)
print(scalar.shape)
print(scalar.item())
tensor(7)
0
torch.Size([])
7

# Vector
vector = torch.tensor([5, 8])
print(vector)
print(vector.ndim)
print(vector.shape)
tensor([5, 8])
1
torch.Size([2])

# Matrix
MATRIX = torch.tensor([[4, 7, 8],
                       [8, 9, 9],
                       [8, 9, 9],
                       [8, 9, 9]])
print(MATRIX)
print(MATRIX.ndim)
print(MATRIX.shape)
tensor([[4, 7, 8],
        [8, 9, 9],
        [8, 9, 9],
        [8, 9, 9]])
2
torch.Size([4, 3])

# Tensor
TENSOR = torch.tensor([[
    [1, 2, 3, 5, 6],
    [3, 5, 7, 6,7]],[
    [1, 2, 3, 5, 6],
    [3, 5, 7, 6,7]],[
    [1, 2, 3, 5, 6],
    [3, 5, 7, 6,7]],[
    [1, 2, 3, 5, 6],
    [3, 5, 7, 6,7]]])
print(TENSOR)
print(TENSOR.ndim)
print(TENSOR.shape)
tensor([[[1, 2, 3, 5, 6],
         [3, 5, 7, 6, 7]],

[[1, 2, 3, 5, 6],
[3, 5, 7, 6, 7]],


[[1, 2, 3, 5, 6],
[3, 5, 7, 6, 7]],


[[1, 2, 3, 5, 6],
[3, 5, 7, 6, 7]]])
3
torch.Size([4, 2, 5])

TENSOR[0][0][4].item()
6
random_scalar = torch.rand(size=())
print("scalar", random_scalar, random_scalar.dtype, random_scalar.ndim)

random_vector = torch.rand(size=(3,))
print("vector", random_vector, random_vector.dtype, random_vector.ndim)

random_matrix = torch.rand(size=(3,5))
print("matrix", random_matrix, random_matrix.dtype, random_matrix.ndim)

random_tensor = torch.rand(size=(2,3,5))
print("tensor", random_tensor, random_tensor.dtype, random_tensor.ndim)
scalar tensor(0.6644) torch.float32 0
vector tensor([0.5092, 0.3265, 0.3305]) torch.float32 1
matrix tensor([[0.7125, 0.1623, 0.5444, 0.2557, 0.1294],
[0.2828, 0.4090, 0.3717, 0.6384, 0.2324],
[0.0896, 0.3283, 0.9117, 0.8554, 0.8026]]) torch.float32 2
tensor tensor([[[0.0764, 0.7815, 0.1997, 0.7713, 0.6133],
[0.5747, 0.5382, 0.1584, 0.0705, 0.4167],
[0.8590, 0.9426, 0.8894, 0.1421, 0.6264]],


[[0.0267, 0.5761, 0.6108, 0.5112, 0.2113],
[0.5155, 0.3585, 0.1487, 0.6395, 0.1117],
[0.3855, 0.0286, 0.4077, 0.9589, 0.2900]]]) torch.float32 3

# Zeros

zeros = torch.zeros(size=(3,4))
zeros, zeros.dtype
(tensor([[0., 0., 0., 0.],
[0., 0., 0., 0.],
[0., 0., 0., 0.]]),
torch.float32)
# Ones

ones = torch.ones(size=(3,4))
ones, ones.dtype
(tensor([[1., 1., 1., 1.],
[1., 1., 1., 1.],
[1., 1., 1., 1.]]),
torch.float32)
# Range

arange = torch.arange(start=1, end=11, step=1)
print(arange)
print(torch.zeros_like(input=arange))
print(torch.ones_like(input=arange))
tensor([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10])
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])

# dtype, device, requires_grad

float16_tensor = torch.tensor([3, 6, 7], dtype=torch.float16, device=None, requires_grad=False)
float16_tensor, float16_tensor.dtype
(tensor([3., 6., 7.], dtype=torch.float16), torch.float16)
some_tensor = torch.rand(3,4)
some_tensor

print(f"dtype is {some_tensor.dtype}")
print(f"shape is {some_tensor.shape}")
print(f"device is {some_tensor.device}")
dtype is torch.float32
shape is torch.Size([3, 4])
device is cpu

Tensor operations

  • Addition
  • Substraction
  • Multiplication (element-wise)
  • Division
  • Matrix multiplication
tensor = torch.tensor([5,6,7])
tensor_two = torch.tensor([7,8,9])

# tensor and scalar
print("Addition", tensor + 10)
print("Substraction", tensor - 10)
print("Multiplication", tensor  10)
print("Division", tensor / 10)

print()

# tensor and tensor. for this to work both tensors should be same shape
print("Addition", tensor + tensor_two)
print("Substraction", tensor - tensor_two)
print("Multiplication element wise", tensor  tensor_two)
print("Division", tensor / tensor_two)
Addition tensor([15, 16, 17])
Substraction tensor([-5, -4, -3])
Multiplication tensor([50, 60, 70])
Division tensor([0.5000, 0.6000, 0.7000])


Addition tensor([12, 14, 16])
Substraction tensor([-2, -2, -2])
Multiplication element wise tensor([35, 48, 63])
Division tensor([0.7143, 0.7500, 0.7778])

# Matrix multiplication. torch.matmul is same as @

print(torch.matmul(torch.rand(3,6), torch.rand(6,8))) # inner dimensions must match

print(torch.rand(3,6) @ torch.rand(6,8))  # inner dimensions must match

# resulting matrix has shape of outer dimensions (3,8)
tensor([[1.3683, 1.6665, 1.0196, 1.4887, 1.3877, 2.0378, 1.4030, 1.2248],
[1.3261, 1.8816, 1.3489, 1.5845, 1.1351, 1.8082, 1.2104, 0.7441],
[2.0075, 2.5350, 1.1848, 2.0803, 1.9189, 2.2428, 1.4352, 1.4079]])
tensor([[1.7329, 1.2273, 0.7495, 0.7849, 2.1434, 1.7336, 1.4021, 1.8141],
[2.1266, 1.5630, 1.3105, 1.0899, 2.4417, 2.0765, 2.0571, 2.1485],
[2.2360, 1.4263, 0.8052, 0.9888, 2.7599, 2.1536, 2.1972, 2.2021]])

torch.tensor([1,2,3]) @ torch.tensor([1,2,3])
tensor(14)
torch.tensor([[2, 3, 4 ], [3, 4, 5],[5,8,3],[7,4,2]]) @ torch.tensor([[3, 4], [6,7], [5, 7]])
tensor([[44, 57],
[58, 75],
[78, 97],
[55, 70]])
tensor = torch.tensor([
    [2, 3, 4], 
    [3, 4, 5],
    [5, 8, 3],
    [7, 4, 2]
]) 

tensor @ tensor.T
tensor([[29, 38, 46, 34],
[38, 50, 62, 47],
[46, 62, 98, 73],
[34, 47, 73, 69]])
tensor.T
tensor([[2, 3, 5, 7],
[3, 4, 8, 4],
[4, 5, 3, 2]])

inner dimensions must match
the resulting matrix has shape of the outer dimensions

(torch.rand(3,10) @ torch.rand(10,4)).shape
torch.Size([3, 4])

Aggregations

min, max, sum, mean

x = torch.arange(0, 100, 10, dtype=torch.float32)
x.min()
tensor(0.)
x.max()
tensor(90.)
x.mean()
tensor(45.)
x.dtype
torch.float32
x.sum()
tensor(450.)
x.argmin()
tensor(0)
x.argmax()
tensor(9)

Reshaping, stacking, squeezing, unsqueezing tensors

  • Reshaping - reshapes an input tensor to a defined shape
  • View - return a view of an input tensor of certain shape but keep the same memory as the original tensor
  • Stacking - combine multiple tensors on top of each other (vstack) or side by side (hstack)
  • Squeeze - removes all 1 dimensions from a tensor
  • Unsqueeze - add a 1 dimension to a target tensor
  • Permute - returns a view of the input dimensions permuted (swapped) in a certain way
x = torch.arange(1., 11.)
x
tensor([ 1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10.])
# Add extra dimension
x_reshaped = x.reshape(5, 2)
x_reshaped, x_reshaped.shape
(tensor([[ 1.,  2.],
[ 3.,  4.],
[ 5.,  6.],
[ 7.,  8.],
[ 9., 10.]]),
torch.Size([5, 2]))
# change view
z = x.view(5,2)
z, z.shape
(tensor([[ 1.,  2.],
[ 3.,  4.],
[ 5.,  6.],
[ 7.,  8.],
[ 9., 10.]]),
torch.Size([5, 2]))
z[:, 0] = 5
z, x
(tensor([[ 5.,  2.],
[ 5.,  4.],
[ 5.,  6.],
[ 5.,  8.],
[ 5., 10.]]),
tensor([ 5.,  2.,  5.,  4.,  5.,  6.,  5.,  8.,  5., 10.]))
# stack tensors on top
x_stacked = torch.stack([x, x, x, x], dim=1)
x_stacked
tensor([[ 5.,  5.,  5.,  5.],
[ 2.,  2.,  2.,  2.],
[ 5.,  5.,  5.,  5.],
[ 4.,  4.,  4.,  4.],
[ 5.,  5.,  5.,  5.],
[ 6.,  6.,  6.,  6.],
[ 5.,  5.,  5.,  5.],
[ 8.,  8.,  8.,  8.],
[ 5.,  5.,  5.,  5.],
[10., 10., 10., 10.]])
# torch.squeeze()
x_reshaped, x_reshaped.shape
(tensor([[ 5.,  2.],
[ 5.,  4.],
[ 5.,  6.],
[ 5.,  8.],
[ 5., 10.]]),
torch.Size([5, 2]))
ones = torch.ones(size=(1,1,1,1,1,10))
ones
tensor([[[[[[1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]]]]]])
torch.squeeze(ones)
tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])
unsqueezed = torch.unsqueeze(torch.squeeze(ones), dim=(0))
unsqueezed
tensor([[1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]])
# permute - rearanges the dimensions
x_original = torch.rand(224,224, 3)
x_original.shape
torch.Size([224, 224, 3])
x_permuted = x_original.permute(2, 0, 1)
x_permuted.shape
torch.Size([3, 224, 224])
x_original[0,0,0] = 123456
x_permuted[0,0,0]
tensor(123456.)

indexing - selecting data from tensors

x = torch.arange(1,10).reshape(1, 3, 3)
x
tensor([[[1, 2, 3],
[4, 5, 6],
[7, 8, 9]]])
x[0]
tensor([[1, 2, 3],
[4, 5, 6],
[7, 8, 9]])
x[0, 0]
tensor([1, 2, 3])
x[0,2,2]
tensor(9)
x[:, 0]
tensor([[1, 2, 3]])
x[:, :, 1]
tensor([[2, 5, 8]])
x[:, 1, 1]
tensor([5])
x[0, 0, :]
tensor([1, 2, 3])
x[0, :, 2]
tensor([3, 6, 9])

pytorch tensors and numpy

import torch
import numpy as np
array = np.arange(1.0, 8.0)
tensor = torch.from_numpy(array)
array, tensor
(array([1., 2., 3., 4., 5., 6., 7.]),
tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float64))
array[0] = 12345
tensor
tensor([1.2345e+04, 2.0000e+00, 3.0000e+00, 4.0000e+00, 5.0000e+00, 6.0000e+00,
7.0000e+00], dtype=torch.float64)
# tensor to numpy
tensor = torch.ones(7)
numpy_tensor = tensor.numpy()
tensor, numpy_tensor
(tensor([1., 1., 1., 1., 1., 1., 1.]),
array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))
tensor[0] = 2
numpy_tensor
array([2., 1., 1., 1., 1., 1., 1.], dtype=float32)

Reproducibility

torch.rand(3,3)
tensor([[0.3238, 0.1884, 0.7731],
[0.0536, 0.9680, 0.8276],
[0.2384, 0.1532, 0.5796]])
random_tensor_A = torch.rand(3, 4)
random_tensor_B = torch.rand(3, 4)
random_tensor_A, random_tensor_B, random_tensor_A == random_tensor_B
(tensor([[0.9475, 0.4847, 0.0121, 0.7039],
[0.3792, 0.7091, 0.3932, 0.4991],
[0.7266, 0.9926, 0.7688, 0.6884]]),
tensor([[0.7315, 0.4388, 0.8574, 0.3715],
[0.0011, 0.1201, 0.9891, 0.3583],
[0.3731, 0.4029, 0.7550, 0.3202]]),
tensor([[False, False, False, False],
[False, False, False, False],
[False, False, False, False]]))
RANDOM_SEED = 42
torch.manual_seed(RANDOM_SEED)

random_tensor_C = torch.rand(3, 4)

torch.manual_seed(RANDOM_SEED)
random_tensor_D = torch.rand(3, 4)
random_tensor_C, random_tensor_D, random_tensor_C == random_tensor_D
(tensor([[0.8823, 0.9150, 0.3829, 0.9593],
[0.3904, 0.6009, 0.2566, 0.7936],
[0.9408, 0.1332, 0.9346, 0.5936]]),
tensor([[0.8823, 0.9150, 0.3829, 0.9593],
[0.3904, 0.6009, 0.2566, 0.7936],
[0.9408, 0.1332, 0.9346, 0.5936]]),
tensor([[True, True, True, True],
[True, True, True, True],
[True, True, True, True]]))

GPU

! nvidia-smi
/bin/bash: nvidia-smi: command not found

check for gpu access

import torch
torch.cuda.is_available()
False
device = "cuda" if torch.cuda.is_available() else "cpu"
device
'cpu'
torch.cuda.device_count()
0

Putting tensors (and models) on the GPU

tensor = torch.tensor([1, 2, 3])
tensor, tensor.device
(tensor([1, 2, 3]), device(type='cpu'))
tensor_on_gpu = tensor.to(device)
tensor_on_gpu
tensor([1, 2, 3])
tensor_on_gpu.cpu()
tensor([1, 2, 3])

Exercices

# 2
torch.rand(7, 7)
tensor([[0.8694, 0.5677, 0.7411, 0.4294, 0.8854, 0.5739, 0.2666],
[0.6274, 0.2696, 0.4414, 0.2969, 0.8317, 0.1053, 0.2695],
[0.3588, 0.1994, 0.5472, 0.0062, 0.9516, 0.0753, 0.8860],
[0.5832, 0.3376, 0.8090, 0.5779, 0.9040, 0.5547, 0.3423],
[0.6343, 0.3644, 0.7104, 0.9464, 0.7890, 0.2814, 0.7886],
[0.5895, 0.7539, 0.1952, 0.0050, 0.3068, 0.1165, 0.9103],
[0.6440, 0.7071, 0.6581, 0.4913, 0.8913, 0.1447, 0.5315]])
# 3
torch.rand(7, 7) @ torch.rand(1, 7).T
tensor([[1.2748],
[1.1652],
[1.0182],
[1.7959],
[1.6076],
[1.8623],
[0.7118]])
# 4
torch.manual_seed(0)
torch.rand(7, 7) @ torch.rand(1, 7).T
tensor([[1.8542],
[1.9611],
[2.2884],
[3.0481],
[1.7067],
[2.5290],
[1.7989]])
torch.cuda.manual_seed(1234)
t1 = torch.rand(2, 3).to(device)
t2 = torch.rand(2, 3, device=device)
t1,t2
(tensor([[0.5932, 0.1123, 0.1535],
[0.2417, 0.7262, 0.7011]]),
tensor([[0.2038, 0.6511, 0.7745],
[0.4369, 0.5191, 0.6159]]))
t3 = t1@t2.T
t3
tensor([[0.3129, 0.4120],
[1.0651, 0.9143]])
t3.min()
tensor(0.3129)
t3.max()
tensor(1.0651)
t3.argmax()
tensor(2)
t3.argmin()
tensor(0)
torch.manual_seed(7)
t4 = torch.rand(1, 1, 1, 10)
t4.squeeze()
tensor([0.5349, 0.1988, 0.6592, 0.6569, 0.2328, 0.4251, 0.2071, 0.6297, 0.3653,
0.8513])

(Этот же ноутбук на кагле)

Начал первую часть. Первая часть посвящена пайторч воркфлоу.