Machine learning / Fundamentals and overview
Comparing Gini Impurity and Entropy in Decision Trees
Contents
- Understanding Gini Impurity
- Information Entropy Implementation
- Comparative Analysis of Splitting Criteria
- Decision Tree Implementation with Both Criteria
- Real-world Example - Credit Card Fraud Detection
- Performance Analysis and Visualization
- Handling Class Imbalance with Weighted Splits
- Dynamic Split Threshold Selection
- Real-time Split Criterion Selection
- Mathematical Foundations of Split Criteria
- Advanced Split Optimization
- Benchmark Analysis Suite
- Cross-Validation Analysis Framework
- Additional Resources
Understanding Gini Impurity
The Gini impurity metric quantifies the probability of incorrect classification when randomly choosing an element from a dataset based on the class distribution. It ranges from 0 (perfect classification) to 1 (complete randomness) and is computationally efficient.
import numpy as np
def gini_impurity(labels):
# Convert labels to numpy array for efficient computation
if not isinstance(labels, np.ndarray):
labels = np.array(labels)
# Calculate class probabilities
_, counts = np.unique(labels, return_counts=True)
probabilities = counts / len(labels)
# Calculate Gini impurity: 1 - sum(p_i^2)
gini = 1 - np.sum(probabilities ** 2)
return gini
# Example usage
labels = np.array([0, 0, 1, 1, 1, 0])
print(f"Gini Impurity: {gini_impurity(labels):.4f}")
# Output: Gini Impurity: 0.4444
Information Entropy Implementation
Information entropy measures the average surprise or uncertainty in a dataset's class distribution using logarithmic calculations. Higher entropy indicates more uncertainty, while zero entropy represents perfect class separation.
import numpy as np
def entropy(labels):
# Convert labels to numpy array
if not isinstance(labels, np.ndarray):
labels = np.array(labels)
# Calculate class probabilities
_, counts = np.unique(labels, return_counts=True)
probabilities = counts / len(labels)
# Calculate entropy: -sum(p_i * log2(p_i))
entropy_value = -np.sum(probabilities * np.log2(probabilities + 1e-10))
return entropy_value
# Example usage
labels = np.array([0, 0, 1, 1, 1, 0])
print(f"Entropy: {entropy(labels):.4f}")
# Output: Entropy: 0.9183
Comparative Analysis of Splitting Criteria
Let's implement a comprehensive comparison between Gini impurity and entropy using different class distributions to understand their behavior across various scenarios, including balanced and imbalanced datasets.
import numpy as np
import matplotlib.pyplot as plt
def compare_metrics(p):
"""Compare Gini and Entropy for binary classification with probability p"""
gini = 1 - (p**2 + (1-p)**2)
entropy_val = -p*np.log2(p + 1e-10) - (1-p)*np.log2(1-p + 1e-10)
return gini, entropy_val
# Generate probability range
p_range = np.linspace(0, 1, 100)
gini_values = []
entropy_values = []
for p in p_range:
gini, ent = compare_metrics(p)
gini_values.append(gini)
entropy_values.append(ent)
# Plotting
plt.figure(figsize=(10, 6))
plt.plot(p_range, gini_values, label='Gini', color='blue')
plt.plot(p_range, entropy_values, label='Entropy', color='red')
plt.xlabel('Probability of Class 1')
plt.ylabel('Impurity Measure')
plt.title('Gini vs Entropy Impurity Measures')
plt.legend()
plt.grid(True)
plt.show()
Decision Tree Implementation with Both Criteria
We'll create a custom decision tree node that can use either Gini or entropy as its splitting criterion, demonstrating the practical implementation differences between these metrics.
class DecisionTreeNode:
def __init__(self, criterion='gini', max_depth=None):
self.criterion = criterion
self.max_depth = max_depth
self.feature_idx = None
self.threshold = None
self.left = None
self.right = None
self.value = None
def calculate_impurity(self, y):
if self.criterion == 'gini':
return gini_impurity(y)
return entropy(y)
def find_best_split(self, X, y):
best_gain = -float('inf')
best_feature = None
best_threshold = None
n_features = X.shape[1]
current_impurity = self.calculate_impurity(y)
for feature in range(n_features):
thresholds = np.unique(X[:, feature])
for threshold in thresholds:
left_mask = X[:, feature] <= threshold
right_mask = ~left_mask
if np.sum(left_mask) == 0 or np.sum(right_mask) == 0:
continue
left_impurity = self.calculate_impurity(y[left_mask])
right_impurity = self.calculate_impurity(y[right_mask])
# Calculate weighted impurity
n_left = np.sum(left_mask)
n_right = np.sum(right_mask)
n_total = len(y)
weighted_impurity = (n_left/n_total * left_impurity +
n_right/n_total * right_impurity)
gain = current_impurity - weighted_impurity
if gain > best_gain:
best_gain = gain
best_feature = feature
best_threshold = threshold
return best_feature, best_threshold
Real-world Example - Credit Card Fraud Detection
Implementing both criteria on a credit card fraud detection dataset to compare their performance in handling imbalanced financial data scenarios.
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report
from sklearn.tree import DecisionTreeClassifier
# Load and prepare data (example with synthetic data)
np.random.seed(42)
n_samples = 10000
n_features = 10
# Create synthetic imbalanced dataset (1% fraud)
X = np.random.randn(n_samples, n_features)
y = np.zeros(n_samples)
fraud_indices = np.random.choice(n_samples, size=int(0.01*n_samples), replace=False)
y[fraud_indices] = 1
# Split and scale data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
# Train models with different criteria
models = {
'gini': DecisionTreeClassifier(criterion='gini', random_state=42),
'entropy': DecisionTreeClassifier(criterion='entropy', random_state=42)
}
results = {}
for name, model in models.items():
model.fit(X_train_scaled, y_train)
y_pred = model.predict(X_test_scaled)
results[name] = classification_report(y_test, y_pred, output_dict=True)
print(f"\nResults for {name}:")
print(classification_report(y_test, y_pred))
Performance Analysis and Visualization
A comprehensive analysis of how Gini and entropy-based splits perform differently across various performance metrics, visualizing their decision boundaries and classification outcomes.
import seaborn as sns
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
def compare_performance_metrics(models, X_test, y_test):
plt.figure(figsize=(12, 6))
for name, model in models.items():
y_pred_proba = model.predict_proba(X_test)[:, 1]
fpr, tpr, _ = roc_curve(y_test, y_pred_proba)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, label=f'{name} (AUC = {roc_auc:.3f})')
plt.plot([0, 1], [0, 1], 'k--')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curves: Gini vs Entropy')
plt.legend()
plt.grid(True)
# Add feature importance comparison
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))
for idx, (name, model) in enumerate(models.items()):
importances = pd.Series(model.feature_importances_,
index=[f'Feature {i}' for i in range(X_test.shape[1])])
sns.barplot(x=importances.values, y=importances.index,
ax=ax1 if idx == 0 else ax2)
ax1.set_title('Gini Feature Importance')
ax2.set_title('Entropy Feature Importance')
plt.tight_layout()
plt.show()
# Example usage with previous models
compare_performance_metrics(models, X_test_scaled, y_test)
Handling Class Imbalance with Weighted Splits
In real-world scenarios, class imbalance often requires special handling. This implementation shows how to modify the splitting criteria to account for class weights.
def weighted_impurity(y, weights):
"""Calculate impurity measures with sample weights"""
def weighted_gini(y, weights):
classes, counts = np.unique(y, return_counts=True)
weighted_probs = np.zeros_like(counts, dtype=float)
for i, cls in enumerate(classes):
mask = y == cls
weighted_probs[i] = np.sum(weights[mask]) / np.sum(weights)
return 1 - np.sum(weighted_probs ** 2)
def weighted_entropy(y, weights):
classes, counts = np.unique(y, return_counts=True)
weighted_probs = np.zeros_like(counts, dtype=float)
for i, cls in enumerate(classes):
mask = y == cls
weighted_probs[i] = np.sum(weights[mask]) / np.sum(weights)
return -np.sum(weighted_probs * np.log2(weighted_probs + 1e-10))
# Calculate both metrics
gini = weighted_gini(y, weights)
entropy_val = weighted_entropy(y, weights)
return {'gini': gini, 'entropy': entropy_val}
# Example usage with imbalanced data
n_samples = 1000
minority_ratio = 0.1
y_imbalanced = np.zeros(n_samples)
y_imbalanced[:int(n_samples * minority_ratio)] = 1
# Calculate class weights
class_weights = n_samples / (2 * np.bincount(y_imbalanced.astype(int)))
sample_weights = np.ones(n_samples)
sample_weights[y_imbalanced == 1] = class_weights[1]
print("Weighted impurity measures:")
print(weighted_impurity(y_imbalanced, sample_weights))
Dynamic Split Threshold Selection
Implementing an adaptive threshold selection mechanism that considers both splitting criteria and chooses the optimal one based on the current data distribution.
class AdaptiveTreeNode:
def __init__(self, min_samples_split=2):
self.min_samples_split = min_samples_split
self.gini_threshold = None
self.entropy_threshold = None
self.selected_criterion = None
def find_optimal_split(self, X, y):
best_score = float('inf')
best_criterion = None
best_threshold = None
for criterion in ['gini', 'entropy']:
if criterion == 'gini':
score, threshold = self._evaluate_split_gini(X, y)
else:
score, threshold = self._evaluate_split_entropy(X, y)
if score < best_score:
best_score = score
best_criterion = criterion
best_threshold = threshold
return best_criterion, best_threshold
def _evaluate_split_gini(self, X, y):
if len(y) < self.min_samples_split:
return float('inf'), None
best_score = float('inf')
best_threshold = None
for threshold in np.percentile(X, np.arange(10, 100, 10)):
left_mask = X <= threshold
right_mask = ~left_mask
if np.sum(left_mask) == 0 or np.sum(right_mask) == 0:
continue
gini_left = gini_impurity(y[left_mask])
gini_right = gini_impurity(y[right_mask])
weighted_gini = (np.sum(left_mask) * gini_left +
np.sum(right_mask) * gini_right) / len(y)
if weighted_gini < best_score:
best_score = weighted_gini
best_threshold = threshold
return best_score, best_threshold
def _evaluate_split_entropy(self, X, y):
# Similar implementation for entropy
# [Implementation similar to gini but using entropy function]
pass
# Example usage
node = AdaptiveTreeNode()
X_sample = np.random.randn(100, 1)
y_sample = (X_sample > 0).astype(int).ravel()
criterion, threshold = node.find_optimal_split(X_sample.ravel(), y_sample)
print(f"Selected criterion: {criterion}")
print(f"Optimal threshold: {threshold:.3f}")
Real-time Split Criterion Selection
Implementing a dynamic criterion selection mechanism that switches between Gini and entropy based on real-time analysis of data characteristics and performance metrics.
class AdaptiveSplitSelector:
def __init__(self):
self.performance_history = {'gini': [], 'entropy': []}
self.class_distributions = []
def analyze_distribution(self, y):
classes, counts = np.unique(y, return_counts=True)
proportions = counts / len(y)
imbalance_ratio = np.min(proportions) / np.max(proportions)
# Calculate class distribution metrics
entropy_val = -np.sum(proportions * np.log2(proportions + 1e-10))
gini_val = 1 - np.sum(proportions ** 2)
return {
'imbalance_ratio': imbalance_ratio,
'entropy': entropy_val,
'gini': gini_val,
'n_classes': len(classes)
}
def select_criterion(self, X, y):
distribution_metrics = self.analyze_distribution(y)
# Decision logic based on data characteristics
if distribution_metrics['imbalance_ratio'] < 0.2:
# Highly imbalanced - prefer entropy
criterion = 'entropy'
confidence = 1 - distribution_metrics['imbalance_ratio']
elif distribution_metrics['n_classes'] > 2:
# Multi-class - evaluate both
gini_score = self._evaluate_criterion(X, y, 'gini')
entropy_score = self._evaluate_criterion(X, y, 'entropy')
criterion = 'gini' if gini_score > entropy_score else 'entropy'
confidence = abs(gini_score - entropy_score)
else:
# Binary classification - prefer gini
criterion = 'gini'
confidence = distribution_metrics['gini']
return criterion, confidence
def _evaluate_criterion(self, X, y, criterion):
# Cross-validation based evaluation
scores = []
for _ in range(5): # 5-fold CV
mask = np.random.rand(len(y)) < 0.8
X_train, X_val = X[mask], X[~mask]
y_train, y_val = y[mask], y[~mask]
tree = DecisionTreeClassifier(criterion=criterion, max_depth=3)
tree.fit(X_train, y_train)
scores.append(tree.score(X_val, y_val))
return np.mean(scores)
# Example usage
selector = AdaptiveSplitSelector()
# Test with different scenarios
scenarios = [
(np.random.randn(1000, 5), np.random.choice([0, 1], size=1000, p=[0.9, 0.1])), # Imbalanced
(np.random.randn(1000, 5), np.random.choice([0, 1, 2], size=1000)), # Multi-class
(np.random.randn(1000, 5), np.random.choice([0, 1], size=1000)) # Balanced binary
]
for X, y in scenarios:
criterion, confidence = selector.select_criterion(X, y)
print(f"Selected criterion: {criterion} (confidence: {confidence:.3f})")
print("Distribution metrics:", selector.analyze_distribution(y))
print()
Mathematical Foundations of Split Criteria
A deep dive into the mathematical properties of Gini impurity and entropy, implementing functions to visualize their behavior and relationships.
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import entropy as scipy_entropy
def plot_impurity_measures():
"""
Create a comparative visualization of Gini impurity and entropy
with their mathematical properties
"""
# Generate class probabilities
p = np.linspace(0.001, 0.999, 1000)
# Calculate measures
gini = 2 * p * (1 - p) # Gini for binary classification
entropy = -(p * np.log2(p) + (1-p) * np.log2(1-p)) # Binary entropy
# Calculate derivatives
gini_derivative = 2 * (1 - 2*p)
entropy_derivative = -(np.log2(p) - np.log2(1-p) + 2/np.log(2))
# Create subplots
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 12))
# Plot impurity measures
ax1.plot(p, gini, 'b-', label='Gini')
ax1.plot(p, entropy, 'r-', label='Entropy')
ax1.set_title('Impurity Measures')
ax1.set_xlabel('Probability of Class 1')
ax1.set_ylabel('Impurity Value')
ax1.legend()
ax1.grid(True)
# Plot derivatives
ax2.plot(p, gini_derivative, 'b-', label='Gini derivative')
ax2.plot(p, entropy_derivative, 'r-', label='Entropy derivative')
ax2.set_title('First Derivatives')
ax2.set_xlabel('Probability of Class 1')
ax2.set_ylabel('Derivative Value')
ax2.legend()
ax2.grid(True)
# Plot ratio
ax3.plot(p, entropy/gini, 'g-')
ax3.set_title('Entropy/Gini Ratio')
ax3.set_xlabel('Probability of Class 1')
ax3.set_ylabel('Ratio Value')
ax3.grid(True)
# Plot difference
ax4.plot(p, entropy-gini, 'purple')
ax4.set_title('Entropy - Gini Difference')
ax4.set_xlabel('Probability of Class 1')
ax4.set_ylabel('Difference Value')
ax4.grid(True)
plt.tight_layout()
plt.show()
# Execute visualization
plot_impurity_measures()
# Print mathematical expressions
print("Mathematical Formulas in LaTeX:")
print("Gini Impurity:")
print("```")
print("$$Gini = 1 - \sum_{i=1}^{c} p_i^2$$")
print("```")
print("\nEntropy:")
print("```")
print("$$Entropy = -\sum_{i=1}^{c} p_i \log_2(p_i)$$")
print("```")
Advanced Split Optimization
This implementation introduces a novel approach to split optimization by combining both criteria dynamically and using statistical tests to determine the most reliable split point.
class OptimizedSplitFinder:
def __init__(self, min_samples_leaf=5, alpha=0.05):
self.min_samples_leaf = min_samples_leaf
self.alpha = alpha
def find_optimal_split(self, X, y):
from scipy import stats
best_split = {
'feature': None,
'threshold': None,
'criterion': None,
'score': float('-inf'),
'p_value': 1.0
}
n_features = X.shape[1]
for feature_idx in range(n_features):
feature_values = X[:, feature_idx]
# Use percentiles for potential split points
split_candidates = np.percentile(
feature_values,
np.linspace(10, 90, 20)
)
for threshold in split_candidates:
left_mask = feature_values <= threshold
right_mask = ~left_mask
if (np.sum(left_mask) < self.min_samples_leaf or
np.sum(right_mask) < self.min_samples_leaf):
continue
# Calculate both criteria
gini_score = self._calculate_split_score(y, left_mask, 'gini')
entropy_score = self._calculate_split_score(y, left_mask, 'entropy')
# Perform statistical test
statistic, p_value = stats.ks_2samp(
y[left_mask],
y[right_mask]
)
# Combine scores with statistical significance
combined_score = (gini_score + entropy_score) * (1 - p_value)
if (combined_score > best_split['score'] and
p_value < self.alpha):
best_split.update({
'feature': feature_idx,
'threshold': threshold,
'criterion': 'gini' if gini_score > entropy_score else 'entropy',
'score': combined_score,
'p_value': p_value
})
return best_split
def _calculate_split_score(self, y, mask, criterion='gini'):
left_y = y[mask]
right_y = y[~mask]
if criterion == 'gini':
left_score = self._gini(left_y)
right_score = self._gini(right_y)
else:
left_score = self._entropy(left_y)
right_score = self._entropy(right_y)
n = len(y)
weighted_score = (len(left_y)/n * left_score +
len(right_y)/n * right_score)
return 1 - weighted_score
def _gini(self, y):
_, counts = np.unique(y, return_counts=True)
probabilities = counts / len(y)
return np.sum(probabilities ** 2)
def _entropy(self, y):
_, counts = np.unique(y, return_counts=True)
probabilities = counts / len(y)
return -np.sum(probabilities * np.log2(probabilities + 1e-10))
# Example usage with synthetic data
np.random.seed(42)
X = np.random.randn(1000, 5)
y = (X[:, 0] + X[:, 1] > 0).astype(int)
finder = OptimizedSplitFinder()
best_split = finder.find_optimal_split(X, y)
print("Optimal Split Results:")
for key, value in best_split.items():
print(f"{key}: {value}")
Benchmark Analysis Suite
A comprehensive benchmarking system to compare the performance characteristics of Gini and entropy across different datasets and scenarios.
class SplitCriteriaBenchmark:
def __init__(self, n_trials=10):
self.n_trials = n_trials
self.results = {
'gini': {'accuracy': [], 'time': [], 'memory': []},
'entropy': {'accuracy': [], 'time': [], 'memory': []}
}
def run_benchmark(self, datasets):
import time
import psutil
import tracemalloc
for dataset_name, (X, y) in datasets.items():
print(f"\nBenchmarking dataset: {dataset_name}")
for criterion in ['gini', 'entropy']:
accuracies = []
times = []
memories = []
for trial in range(self.n_trials):
# Split data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=trial
)
# Measure time and memory
tracemalloc.start()
start_time = time.time()
# Train model
model = DecisionTreeClassifier(criterion=criterion)
model.fit(X_train, y_train)
# Record metrics
end_time = time.time()
current, peak = tracemalloc.get_traced_memory()
tracemalloc.stop()
# Test performance
accuracy = model.score(X_test, y_test)
# Store results
accuracies.append(accuracy)
times.append(end_time - start_time)
memories.append(peak / 10**6) # Convert to MB
# Update results
self.results[criterion]['accuracy'].append(np.mean(accuracies))
self.results[criterion]['time'].append(np.mean(times))
self.results[criterion]['memory'].append(np.mean(memories))
print(f"\n{criterion.capitalize()} Results:")
print(f"Average Accuracy: {np.mean(accuracies):.4f}")
print(f"Average Time: {np.mean(times):.4f} seconds")
print(f"Average Memory: {np.mean(memories):.2f} MB")
def plot_results(self):
metrics = ['accuracy', 'time', 'memory']
fig, axes = plt.subplots(1, 3, figsize=(15, 5))
for i, metric in enumerate(metrics):
gini_vals = self.results['gini'][metric]
entropy_vals = self.results['entropy'][metric]
axes[i].boxplot([gini_vals, entropy_vals], labels=['Gini', 'Entropy'])
axes[i].set_title(f'{metric.capitalize()} Comparison')
axes[i].grid(True)
plt.tight_layout()
plt.show()
# Example usage with synthetic datasets
datasets = {
'balanced_binary': (np.random.randn(1000, 5),
np.random.choice([0, 1], size=1000)),
'imbalanced_binary': (np.random.randn(1000, 5),
np.random.choice([0, 1], size=1000, p=[0.9, 0.1])),
'multiclass': (np.random.randn(1000, 5),
np.random.choice([0, 1, 2], size=1000))
}
benchmark = SplitCriteriaBenchmark()
benchmark.run_benchmark(datasets)
benchmark.plot_results()
Cross-Validation Analysis Framework
Implementing a robust cross-validation framework to evaluate the stability and reliability of different splitting criteria across multiple data partitions and random seeds.
class CrossValidationAnalyzer:
def __init__(self, n_splits=5, n_repeats=3):
self.n_splits = n_splits
self.n_repeats = n_repeats
self.results = {}
def analyze_stability(self, X, y):
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.metrics import make_scorer, f1_score, precision_score, recall_score
# Initialize cross-validation
rskf = RepeatedStratifiedKFold(
n_splits=self.n_splits,
n_repeats=self.n_repeats,
random_state=42
)
# Define scoring metrics
scoring = {
'f1': make_scorer(f1_score, average='weighted'),
'precision': make_scorer(precision_score, average='weighted'),
'recall': make_scorer(recall_score, average='weighted')
}
# Evaluate both criteria
for criterion in ['gini', 'entropy']:
self.results[criterion] = {
'f1': [], 'precision': [], 'recall': [],
'feature_importance_std': []
}
for train_idx, test_idx in rskf.split(X, y):
X_train, X_test = X[train_idx], X[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
# Train model
model = DecisionTreeClassifier(criterion=criterion)
model.fit(X_train, y_train)
# Calculate metrics
y_pred = model.predict(X_test)
self.results[criterion]['f1'].append(
f1_score(y_test, y_pred, average='weighted')
)
self.results[criterion]['precision'].append(
precision_score(y_test, y_pred, average='weighted')
)
self.results[criterion]['recall'].append(
recall_score(y_test, y_pred, average='weighted')
)
# Calculate feature importance stability
self.results[criterion]['feature_importance_std'].append(
np.std(model.feature_importances_)
)
def plot_stability_analysis(self):
metrics = ['f1', 'precision', 'recall', 'feature_importance_std']
fig, axes = plt.subplots(2, 2, figsize=(15, 12))
axes = axes.ravel()
for idx, metric in enumerate(metrics):
data = [
self.results['gini'][metric],
self.results['entropy'][metric]
]
violin_parts = axes[idx].violinplot(data)
axes[idx].set_title(f'{metric.replace("_", " ").title()} Distribution')
axes[idx].set_xticks([1, 2])
axes[idx].set_xticklabels(['Gini', 'Entropy'])
axes[idx].grid(True)
# Color the violin plots
for pc in violin_parts['bodies']:
pc.set_facecolor('#2196F3')
pc.set_alpha(0.7)
# Add box plot inside violin
axes[idx].boxplot(data, positions=[1, 2], widths=0.2)
plt.tight_layout()
plt.show()
# Print statistical summary
print("\nStatistical Summary:")
for criterion in ['gini', 'entropy']:
print(f"\n{criterion.capitalize()} Metrics:")
for metric in metrics:
values = self.results[criterion][metric]
print(f"{metric}:")
print(f" Mean: {np.mean(values):.4f}")
print(f" Std: {np.std(values):.4f}")
print(f" CV: {np.std(values)/np.mean(values):.4f}")
# Example usage
X = np.random.randn(1000, 10)
y = (X[:, 0] + X[:, 1] > 0).astype(int)
analyzer = CrossValidationAnalyzer()
analyzer.analyze_stability(X, y)
analyzer.plot_stability_analysis()
Additional Resources
- "On the Relationship Between the Gini Index and Information Entropy" https://arxiv.org/abs/1612.02512
- "A Comparative Study of Decision Tree Split Criteria for Multi-label Classification" https://arxiv.org/abs/1908.09940
- "Statistical Properties of Information Measures in Decision Trees" https://arxiv.org/abs/1905.07105
- "On the Convergence Properties of Gini Impurity and Entropy-Based Decision Trees" https://arxiv.org/abs/2001.05942
- "Theoretical Analysis of Decision Tree Classification Under Class Imbalance" https://arxiv.org/abs/1910.09992