Implementation of basic automatic differentiation concepts used in deep learning frameworks.
import numpy as np
class Tensor:
def __init__(self, data, requires_grad=False):
self.data = np.array(data)
self.requires_grad = requires_grad
self.grad = None
self._backward = lambda: None
self._prev = set()
def backward(self, gradient=None):
if gradient is None:
gradient = np.ones_like(self.data)
self.grad = gradient
self._backward()
for prev in self._prev:
if prev.requires_grad:
prev.backward(prev.grad)
def sigmoid(x):
return 1 / (1 + np.exp(-x))
class Layer:
def __init__(self, in_features, out_features):
self.weights = Tensor(
np.random.randn(in_features, out_features) * 0.01,
requires_grad=True
)
self.bias = Tensor(
np.zeros((1, out_features)),
requires_grad=True
)
def forward(self, x):
# Linear transformation: y = wx + b
out = Tensor(np.dot(x.data, self.weights.data) + self.bias.data)
def _backward():
if self.weights.requires_grad:
self.weights.grad = np.dot(x.data.T, out.grad)
if self.bias.requires_grad:
self.bias.grad = np.sum(out.grad, axis=0, keepdims=True)
if x.requires_grad:
x.grad = np.dot(out.grad, self.weights.data.T)
out._backward = _backward
out._prev = {self.weights, self.bias, x}
return out
# Example usage
if __name__ == "__main__":
# Create a simple neural network layer
layer = Layer(3, 2)
# Forward pass
x = Tensor(np.random.randn(1, 3), requires_grad=True)
out = layer.forward(x)
# Backward pass (compute gradients)
out.backward(np.array([[1.0, 1.0]]))
print("Input gradient shape:", x.grad.shape)
print("Weight gradient shape:", layer.weights.grad.shape)
print("Bias gradient shape:", layer.bias.grad.shape)