Machine learning / Regularization overfitting and optimization
Diagnosing Overfitting in Machine Learning Models
Contents
- Understanding Overfitting in Machine Learning
- Implementing Cross-Validation for Overfitting Detection
- Learning Curves Analysis
- Regularization Techniques
- Early Stopping Implementation
- Dropout Layer Implementation
- Model Complexity Analysis
- Real-world Example: Credit Card Fraud Detection
- Feature Selection for Overfitting Prevention
- Ensemble Methods to Reduce Overfitting
- Data Augmentation Techniques
- Model Validation Pipeline
- Real-world Example: Image Classification with Regularization
- Bias-Variance Decomposition Analysis
- Additional Resources
Understanding Overfitting in Machine Learning
In machine learning, high training accuracy coupled with poor test accuracy is a classic symptom of overfitting. This occurs when a model learns the training data too perfectly, memorizing noise and specific patterns that don't generalize well to unseen data.
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.metrics import accuracy_score
# Generate synthetic data
np.random.seed(42)
X = np.linspace(0, 1, 30).reshape(-1, 1)
y = np.sin(2 * np.pi * X) + np.random.normal(0, 0.2, X.shape)
# Create and train models with different complexities
degrees = [1, 15] # Linear vs High-degree polynomial
for degree in degrees:
poly = PolynomialFeatures(degree=degree)
X_poly = poly.fit_transform(X)
model = LinearRegression()
model.fit(X_poly, y)
# Calculate training accuracy
y_pred = model.predict(X_poly)
print(f"Degree {degree} - Training MSE:",
np.mean((y - y_pred) ** 2))
Implementing Cross-Validation for Overfitting Detection
Cross-validation helps identify overfitting by evaluating model performance on multiple different train-test splits. This systematic approach provides a more robust estimate of model generalization capability and helps detect overfitting early.
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
import numpy as np
def cross_validate_complexity(X, y, max_degree=15):
kf = KFold(n_splits=5, shuffle=True, random_state=42)
degrees = range(1, max_degree + 1)
train_scores = []
val_scores = []
for degree in degrees:
poly = PolynomialFeatures(degree=degree)
cv_train_scores = []
cv_val_scores = []
for train_idx, val_idx in kf.split(X):
X_train, X_val = X[train_idx], X[val_idx]
y_train, y_val = y[train_idx], y[val_idx]
# Transform features
X_train_poly = poly.fit_transform(X_train)
X_val_poly = poly.transform(X_val)
# Train model
model = LinearRegression()
model.fit(X_train_poly, y_train)
# Calculate scores
train_score = mean_squared_error(y_train,
model.predict(X_train_poly))
val_score = mean_squared_error(y_val,
model.predict(X_val_poly))
cv_train_scores.append(train_score)
cv_val_scores.append(val_score)
train_scores.append(np.mean(cv_train_scores))
val_scores.append(np.mean(cv_val_scores))
return train_scores, val_scores
Learning Curves Analysis
Learning curves provide visual insight into model performance by plotting training and validation errors against training set size. A widening gap between training and validation performance is a clear indicator of overfitting.
def plot_learning_curves(X, y, model, train_sizes=np.linspace(0.1, 1.0, 10)):
from sklearn.model_selection import learning_curve
train_sizes, train_scores, val_scores = learning_curve(
model, X, y,
train_sizes=train_sizes,
cv=5,
scoring='neg_mean_squared_error',
n_jobs=-1
)
# Calculate mean and std
train_mean = -train_scores.mean(axis=1)
train_std = train_scores.std(axis=1)
val_mean = -val_scores.mean(axis=1)
val_std = val_scores.std(axis=1)
plt.figure(figsize=(10, 6))
plt.plot(train_sizes, train_mean, label='Training error')
plt.plot(train_sizes, val_mean, label='Validation error')
plt.fill_between(train_sizes,
train_mean - train_std,
train_mean + train_std,
alpha=0.1)
plt.fill_between(train_sizes,
val_mean - val_std,
val_mean + val_std,
alpha=0.1)
plt.xlabel('Training Set Size')
plt.ylabel('Mean Squared Error')
plt.title('Learning Curves')
plt.legend()
plt.grid(True)
plt.show()
Regularization Techniques
Regularization helps prevent overfitting by adding penalties to the model's complexity. L1 and L2 regularization are common techniques that constrain model parameters, encouraging simpler models that generalize better to unseen data.
from sklearn.linear_model import Ridge, Lasso
import numpy as np
def compare_regularization(X, y, alphas=[0.0001, 0.001, 0.01, 0.1, 1, 10]):
# Split data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42
)
results = []
for alpha in alphas:
# L2 regularization (Ridge)
ridge = Ridge(alpha=alpha)
ridge.fit(X_train, y_train)
ridge_train = mean_squared_error(y_train,
ridge.predict(X_train))
ridge_test = mean_squared_error(y_test,
ridge.predict(X_test))
# L1 regularization (Lasso)
lasso = Lasso(alpha=alpha)
lasso.fit(X_train, y_train)
lasso_train = mean_squared_error(y_train,
lasso.predict(X_train))
lasso_test = mean_squared_error(y_test,
lasso.predict(X_test))
results.append({
'alpha': alpha,
'ridge_train': ridge_train,
'ridge_test': ridge_test,
'lasso_train': lasso_train,
'lasso_test': lasso_test
})
return results
Early Stopping Implementation
Early stopping prevents overfitting by monitoring validation performance during training and stopping when performance starts to degrade. This technique is particularly useful in neural networks and gradient boosting models.
import numpy as np
from sklearn.base import BaseEstimator, RegressorMixin
class EarlyStoppingRegressor(BaseEstimator, RegressorMixin):
def __init__(self, estimator, patience=5, min_delta=0):
self.estimator = estimator
self.patience = patience
self.min_delta = min_delta
def fit(self, X, y, X_val, y_val):
best_val_loss = float('inf')
patience_counter = 0
self.training_history = []
for epoch in range(1000): # Maximum iterations
# Train one epoch
self.estimator.partial_fit(X, y)
# Calculate validation loss
val_loss = mean_squared_error(
y_val,
self.estimator.predict(X_val)
)
self.training_history.append(val_loss)
# Check for improvement
if val_loss < (best_val_loss - self.min_delta):
best_val_loss = val_loss
patience_counter = 0
else:
patience_counter += 1
# Early stopping check
if patience_counter >= self.patience:
print(f"Early stopping at epoch {epoch}")
break
return self
def predict(self, X):
return self.estimator.predict(X)
Dropout Layer Implementation
Dropout is a powerful regularization technique that randomly deactivates neurons during training, preventing co-adaptation of features. This implementation demonstrates how dropout works in practice and its effect on model generalization.
import numpy as np
class DropoutLayer:
def __init__(self, dropout_rate=0.5):
self.dropout_rate = dropout_rate
self.mask = None
def forward(self, X, training=True):
if training:
# Create dropout mask
self.mask = np.random.binomial(1, (1 - self.dropout_rate),
size=X.shape) / (1 - self.dropout_rate)
# Apply mask
return X * self.mask
return X
def backward(self, dout):
# Backward pass applies same mask
return dout * self.mask
# Example usage
X = np.random.randn(100, 20) # 100 samples, 20 features
dropout = DropoutLayer(dropout_rate=0.3)
# Training phase
X_train = dropout.forward(X, training=True)
print(f"Training phase - Active neurons: {np.mean(X_train != 0):.2%}")
# Inference phase
X_test = dropout.forward(X, training=False)
print(f"Testing phase - Active neurons: {np.mean(X_test != 0):.2%}")
Model Complexity Analysis
Understanding the relationship between model complexity and generalization performance is crucial. This implementation provides tools to analyze how different model architectures affect the bias-variance tradeoff.
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPRegressor
import numpy as np
import matplotlib.pyplot as plt
def analyze_model_complexity(X, y, hidden_layers_range):
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42
)
train_scores = []
test_scores = []
for hidden_units in hidden_layers_range:
# Create model with varying complexity
model = MLPRegressor(
hidden_layer_sizes=(hidden_units,),
max_iter=1000,
random_state=42
)
# Train and evaluate
model.fit(X_train, y_train)
train_score = model.score(X_train, y_train)
test_score = model.score(X_test, y_test)
train_scores.append(train_score)
test_scores.append(test_score)
print(f"Hidden units: {hidden_units}")
print(f"Train R²: {train_score:.4f}")
print(f"Test R²: {test_score:.4f}\n")
return train_scores, test_scores
# Example usage
hidden_layers = [2, 4, 8, 16, 32, 64, 128]
train_r2, test_r2 = analyze_model_complexity(X, y, hidden_layers)
Real-world Example: Credit Card Fraud Detection
In this practical example, we'll implement a fraud detection model and demonstrate how to handle overfitting in an imbalanced dataset scenario using various techniques discussed earlier.
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
from imblearn.over_sampling import SMOTE
def fraud_detection_pipeline(X, y):
# Split data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42
)
# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
# Handle imbalanced data
smote = SMOTE(random_state=42)
X_train_balanced, y_train_balanced = smote.fit_resample(
X_train_scaled, y_train
)
# Train model with cross-validation
model = RandomForestClassifier(
n_estimators=100,
max_depth=10,
min_samples_split=10,
random_state=42
)
# Fit and predict
model.fit(X_train_balanced, y_train_balanced)
y_pred = model.predict(X_test_scaled)
# Print results
print(classification_report(y_test, y_pred))
return model, scaler
Feature Selection for Overfitting Prevention
Feature selection helps reduce overfitting by eliminating irrelevant or redundant features. This implementation shows various feature selection methods and their impact on model performance.
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.feature_selection import RFE
from sklearn.ensemble import RandomForestClassifier
class FeatureSelector:
def __init__(self, n_features):
self.n_features = n_features
def statistical_selection(self, X, y):
selector = SelectKBest(score_func=f_classif, k=self.n_features)
X_selected = selector.fit_transform(X, y)
selected_features = selector.get_support()
return X_selected, selected_features
def recursive_elimination(self, X, y):
estimator = RandomForestClassifier(n_estimators=100, random_state=42)
selector = RFE(estimator=estimator, n_features_to_select=self.n_features)
X_selected = selector.fit_transform(X, y)
selected_features = selector.support_
return X_selected, selected_features
# Usage example
selector = FeatureSelector(n_features=10)
X_statistical, stat_features = selector.statistical_selection(X, y)
X_recursive, rec_features = selector.recursive_elimination(X, y)
print("Statistical Selection Shape:", X_statistical.shape)
print("Recursive Elimination Shape:", X_recursive.shape)
Ensemble Methods to Reduce Overfitting
Ensemble methods combine multiple models to create a more robust predictor. This implementation shows how bagging and boosting techniques can help reduce overfitting by averaging out individual model biases.
from sklearn.ensemble import BaggingRegressor, GradientBoostingRegressor
from sklearn.tree import DecisionTreeRegressor
import numpy as np
class OverfitPreventingEnsemble:
def __init__(self, n_estimators=50):
self.n_estimators = n_estimators
def create_ensemble(self, X, y):
# Create base models
bagging = BaggingRegressor(
base_estimator=DecisionTreeRegressor(max_depth=3),
n_estimators=self.n_estimators,
random_state=42
)
boosting = GradientBoostingRegressor(
n_estimators=self.n_estimators,
learning_rate=0.1,
max_depth=3,
random_state=42
)
# Train models
X_train, X_val, y_train, y_val = train_test_split(
X, y, test_size=0.2, random_state=42
)
bagging.fit(X_train, y_train)
boosting.fit(X_train, y_train)
# Evaluate
bagging_score = bagging.score(X_val, y_val)
boosting_score = boosting.score(X_val, y_val)
print(f"Bagging R² Score: {bagging_score:.4f}")
print(f"Boosting R² Score: {boosting_score:.4f}")
return bagging, boosting
# Example usage
ensemble = OverfitPreventingEnsemble()
bagging_model, boosting_model = ensemble.create_ensemble(X, y)
Data Augmentation Techniques
Data augmentation helps prevent overfitting by artificially increasing the training set size through controlled transformations. This implementation demonstrates various augmentation techniques for different types of data.
import numpy as np
from scipy.ndimage import rotate, zoom
class DataAugmenter:
def __init__(self, noise_level=0.05, rotation_range=20):
self.noise_level = noise_level
self.rotation_range = rotation_range
def add_gaussian_noise(self, X):
noise = np.random.normal(0, self.noise_level, X.shape)
return X + noise
def random_rotation(self, X):
angle = np.random.uniform(-self.rotation_range, self.rotation_range)
if len(X.shape) == 2:
return rotate(X, angle, reshape=False)
return np.array([rotate(x, angle, reshape=False) for x in X])
def random_scaling(self, X, scale_range=(0.8, 1.2)):
scale = np.random.uniform(*scale_range)
if len(X.shape) == 2:
return zoom(X, scale)
return np.array([zoom(x, scale) for x in X])
def augment_dataset(self, X, augmentation_factor=2):
augmented_data = [X]
for _ in range(augmentation_factor - 1):
# Apply random combination of augmentations
aug_X = X.copy()
if np.random.random() > 0.5:
aug_X = self.add_gaussian_noise(aug_X)
if np.random.random() > 0.5:
aug_X = self.random_rotation(aug_X)
if np.random.random() > 0.5:
aug_X = self.random_scaling(aug_X)
augmented_data.append(aug_X)
return np.concatenate(augmented_data, axis=0)
# Example usage
augmenter = DataAugmenter()
X_augmented = augmenter.augment_dataset(X)
print(f"Original dataset size: {len(X)}")
print(f"Augmented dataset size: {len(X_augmented)}")
Model Validation Pipeline
A comprehensive validation pipeline is essential to detect and prevent overfitting. This implementation creates a robust pipeline that combines multiple validation techniques.
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import make_scorer, mean_squared_error
import numpy as np
class ValidationPipeline:
def __init__(self, model, n_splits=5):
self.model = model
self.n_splits = n_splits
self.kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)
def validate_model(self, X, y):
# Cross-validation scores
cv_scores = cross_val_score(
self.model, X, y,
cv=self.kf,
scoring=make_scorer(mean_squared_error)
)
# Hold-out validation
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42
)
self.model.fit(X_train, y_train)
train_score = mean_squared_error(
y_train, self.model.predict(X_train)
)
test_score = mean_squared_error(
y_test, self.model.predict(X_test)
)
results = {
'cv_mean_mse': np.mean(cv_scores),
'cv_std_mse': np.std(cv_scores),
'train_mse': train_score,
'test_mse': test_score,
'overfitting_ratio': test_score / train_score
}
return results
# Example usage
from sklearn.ensemble import RandomForestRegressor
pipeline = ValidationPipeline(
RandomForestRegressor(random_state=42)
)
validation_results = pipeline.validate_model(X, y)
Real-world Example: Image Classification with Regularization
This implementation demonstrates how to prevent overfitting in a convolutional neural network for image classification using multiple regularization techniques simultaneously.
import tensorflow as tf
from tensorflow.keras import layers, models, regularizers
class RegularizedCNN:
def __init__(self, input_shape, num_classes):
self.input_shape = input_shape
self.num_classes = num_classes
def build_model(self, dropout_rate=0.5, l2_lambda=0.01):
model = models.Sequential([
# Convolutional layers with L2 regularization
layers.Conv2D(32, (3, 3), activation='relu',
input_shape=self.input_shape,
kernel_regularizer=regularizers.l2(l2_lambda)),
layers.MaxPooling2D((2, 2)),
layers.Dropout(dropout_rate/2),
layers.Conv2D(64, (3, 3), activation='relu',
kernel_regularizer=regularizers.l2(l2_lambda)),
layers.MaxPooling2D((2, 2)),
layers.Dropout(dropout_rate/2),
layers.Conv2D(64, (3, 3), activation='relu',
kernel_regularizer=regularizers.l2(l2_lambda)),
layers.Flatten(),
layers.Dense(64, activation='relu',
kernel_regularizer=regularizers.l2(l2_lambda)),
layers.Dropout(dropout_rate),
layers.Dense(self.num_classes, activation='softmax')
])
return model
def train_model(self, X_train, y_train, X_val, y_val):
model = self.build_model()
# Early stopping callback
early_stopping = tf.keras.callbacks.EarlyStopping(
monitor='val_loss',
patience=5,
restore_best_weights=True
)
# Learning rate reduction callback
lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.2,
patience=3
)
# Compile and train
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
history = model.fit(
X_train, y_train,
epochs=50,
validation_data=(X_val, y_val),
callbacks=[early_stopping, lr_reducer],
batch_size=32
)
return model, history
# Example usage
cnn = RegularizedCNN(input_shape=(28, 28, 1), num_classes=10)
model, history = cnn.train_model(X_train, y_train, X_val, y_val)
Bias-Variance Decomposition Analysis
This implementation provides tools to analyze the bias-variance tradeoff in your models, helping identify whether overfitting is caused by high variance or model complexity issues.
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.base import clone
class BiasVarianceDecomposition:
def __init__(self, base_model, n_bootstrap=100):
self.base_model = base_model
self.n_bootstrap = n_bootstrap
def bootstrap_predictions(self, X_train, y_train, X_test):
predictions = np.zeros((self.n_bootstrap, len(X_test)))
for i in range(self.n_bootstrap):
# Bootstrap sample
indices = np.random.choice(
len(X_train),
size=len(X_train),
replace=True
)
X_boot = X_train[indices]
y_boot = y_train[indices]
# Train model and predict
model = clone(self.base_model)
model.fit(X_boot, y_boot)
predictions[i] = model.predict(X_test)
return predictions
def compute_metrics(self, X, y, test_size=0.2):
# Split data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=test_size, random_state=42
)
# Get bootstrap predictions
predictions = self.bootstrap_predictions(X_train, y_train, X_test)
# Calculate components
expected_predictions = np.mean(predictions, axis=0)
bias = np.mean((y_test - expected_predictions) ** 2)
variance = np.mean(np.var(predictions, axis=0))
total_error = bias + variance
return {
'bias': bias,
'variance': variance,
'total_error': total_error,
'bias_ratio': bias/total_error,
'variance_ratio': variance/total_error
}
# Example usage
from sklearn.tree import DecisionTreeRegressor
decomposer = BiasVarianceDecomposition(
DecisionTreeRegressor(random_state=42)
)
metrics = decomposer.compute_metrics(X, y)
Additional Resources
- High-Bias, Low-Variance Introduction to Machine Learning
- Understanding Deep Learning Requires Rethinking Generalization
- An Overview of Regularization Techniques in Deep Learning
- Practical Guidelines for Preventing Overfitting in Machine Learning Models
- A Survey of Cross-Validation Procedures for Model Selection