Regularization techniques help prevent overfitting in neural networks by constraining model complexity and introducing beneficial biases. These methods improve model generalization and robustness.
Key aspects:
Methods that constrain the model's parameter values:
Adding noise during training to improve robustness:
Artificially increasing training data diversity:
Implementation of regularization from scratch to understand the process:
import numpy as np
class RegularizedNeuralNetwork:
def __init__(self, layer_sizes, l1=0.0, l2=0.0):
self.weights = []
self.biases = []
self.l1 = l1
self.l2 = l2
for i in range(len(layer_sizes) - 1):
self.weights.append(np.random.randn(layer_sizes[i], layer_sizes[i+1]) * 0.01)
self.biases.append(np.zeros((1, layer_sizes[i+1])))
def sigmoid(self, x):
return 1 / (1 + np.exp(-x))
def sigmoid_derivative(self, x):
s = self.sigmoid(x)
return s * (1 - s)
def forward_propagation(self, X):
self.activations = [X]
self.z_values = []
activation = X
for W, b in zip(self.weights, self.biases):
z = np.dot(activation, W) + b
self.z_values.append(z)
activation = self.sigmoid(z)
self.activations.append(activation)
return activation
def compute_loss(self, output, y):
m = y.shape[0]
mse = np.mean(np.square(output - y))
l1_penalty = self.l1 * sum(np.sum(np.abs(W)) for W in self.weights)
l2_penalty = self.l2 * sum(np.sum(W**2) for W in self.weights)
return mse + l1_penalty + l2_penalty
def backward_propagation(self, X, y, learning_rate=0.1):
m = X.shape[0]
delta = self.activations[-1] - y
dW = []
db = []
for l in reversed(range(len(self.weights))):
dW_l = np.dot(self.activations[l].T, delta) / m
db_l = np.sum(delta, axis=0, keepdims=True) / m
# Add regularization gradients
dW_l += self.l1 * np.sign(self.weights[l]) + 2 * self.l2 * self.weights[l]
dW.insert(0, dW_l)
db.insert(0, db_l)
if l > 0:
delta = np.dot(delta, self.weights[l].T) * self.sigmoid_derivative(self.z_values[l-1])
for l in range(len(self.weights)):
self.weights[l] -= learning_rate * dW[l]
self.biases[l] -= learning_rate * db[l]
def train(self, X, y, epochs=1000, learning_rate=0.1):
for epoch in range(epochs):
output = self.forward_propagation(X)
loss = self.compute_loss(output, y)
self.backward_propagation(X, y, learning_rate)
if epoch % 100 == 0:
print(f"Epoch {epoch}, Loss: {loss:.4f}")
# Example usage
if __name__ == "__main__":
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([[0], [1], [1], [0]])
nn = RegularizedNeuralNetwork([2, 4, 1], l1=0.01, l2=0.01)
# nn.train(X, y) # Uncomment to train
Can you explain how regularization works and why it's important?