Deep learning / Regularization and training tricks
Scaling Deep Neural Networks with GPipe in Python
Contents
- Introduction to GPipe
- Micro-Batch Pipeline Parallelism
- Model Partitioning
- Automatic Gradient Accumulation
- Optimizing Communication
- Memory Efficiency
- Handling Non-Uniform Partitions
- Synchronization and Consistency
- Handling Variable-Length Sequences
- Checkpointing and Resuming Training
- Real-Life Example: Language Model Training
- Real-Life Example: Image Segmentation
- Handling Dynamic Computation Graphs
- Performance Optimization Techniques
- Additional Resources
Introduction to GPipe
GPipe is a scalable pipeline parallelism library for training large deep neural networks. It enables efficient training of models that are too large to fit on a single accelerator by partitioning the model across multiple devices and leveraging pipeline parallelism.
import GPipe
import torch.nn as nn
class LargeModel(nn.Module):
def __init__(self):
super().__init__()
self.layers = nn.Sequential(*[nn.Linear(1000, 1000) for _ in range(100)])
def forward(self, x):
return self.layers(x)
model = LargeModel()
model = GPipe(model, balance=[10] * 10, chunks=8)
Micro-Batch Pipeline Parallelism
GPipe implements micro-batch pipeline parallelism, which divides the input mini-batch into smaller micro-batches. These micro-batches are then processed through the pipeline, allowing for efficient utilization of multiple devices.
def train_with_gpipe(model, dataloader, optimizer, num_micro_batches):
for batch in dataloader:
optimizer.zero_grad()
micro_batches = torch.chunk(batch, num_micro_batches)
outputs = []
for micro_batch in micro_batches:
output = model(micro_batch)
outputs.append(output)
loss = torch.cat(outputs).mean()
loss.backward()
optimizer.step()
Model Partitioning
GPipe automatically partitions the model across available devices, balancing the computational load. This partitioning is done based on the number of parameters or the estimated computational cost of each layer.
def partition_model(model, num_partitions):
total_params = sum(p.numel() for p in model.parameters())
params_per_partition = total_params // num_partitions
current_partition = []
current_params = 0
partitions = []
for layer in model.children():
layer_params = sum(p.numel() for p in layer.parameters())
if current_params + layer_params > params_per_partition and current_partition:
partitions.append(nn.Sequential(*current_partition))
current_partition = []
current_params = 0
current_partition.append(layer)
current_params += layer_params
if current_partition:
partitions.append(nn.Sequential(*current_partition))
return partitions
model = LargeModel()
partitioned_model = partition_model(model, num_partitions=4)
Automatic Gradient Accumulation
GPipe handles gradient accumulation automatically, ensuring that gradients are correctly accumulated across micro-batches and devices. This simplifies the training process and maintains consistency with non-pipelined training.
class GPipeModule(nn.Module):
def __init__(self, partitions):
super().__init__()
self.partitions = nn.ModuleList(partitions)
def forward(self, x):
for partition in self.partitions:
x = partition(x)
return x
def backward(self, grad_output):
for partition in reversed(self.partitions):
grad_output = partition.backward(grad_output)
return grad_output
model = GPipeModule(partitioned_model)
Optimizing Communication
GPipe optimizes communication between devices by overlapping computation and communication. This is achieved through careful scheduling of forward and backward passes across the pipeline stages.
import torch.distributed as dist
def optimize_communication(partitioned_model):
for i, partition in enumerate(partitioned_model):
device = f"cuda:{i}"
partition.to(device)
def forward_backward_pass(input_data):
outputs = []
for i, partition in enumerate(partitioned_model):
if i > 0:
input_data = dist.recv(tensor=input_data, src=i-1)
output = partition(input_data)
if i < len(partitioned_model) - 1:
dist.send(tensor=output, dst=i+1)
outputs.append(output)
grad_output = torch.ones_like(outputs[-1])
for i, partition in reversed(list(enumerate(partitioned_model))):
if i < len(partitioned_model) - 1:
grad_output = dist.recv(tensor=grad_output, src=i+1)
grad_input = partition.backward(grad_output)
if i > 0:
dist.send(tensor=grad_input, dst=i-1)
return forward_backward_pass
Memory Efficiency
GPipe achieves memory efficiency by releasing the memory of intermediate activations as soon as they are no longer needed. This allows for training larger models with limited memory resources.
class MemoryEfficientGPipeModule(nn.Module):
def __init__(self, partitions):
super().__init__()
self.partitions = nn.ModuleList(partitions)
def forward(self, x):
activations = []
for partition in self.partitions:
x = partition(x)
activations.append(x.detach().requires_grad_())
return activations[-1], activations[:-1]
def backward(self, grad_output, saved_activations):
for partition, activation in zip(reversed(self.partitions[1:]), reversed(saved_activations)):
grad_output = partition.backward(grad_output)
grad_output = grad_output + activation.grad
activation.grad = None # Free memory
grad_output = self.partitions[0].backward(grad_output)
return grad_output
model = MemoryEfficientGPipeModule(partitioned_model)
Handling Non-Uniform Partitions
GPipe can handle non-uniform partitions, where different parts of the model have varying computational requirements. This is particularly useful for models with heterogeneous architectures.
def create_non_uniform_partitions(model, partition_sizes):
partitions = []
layer_index = 0
for size in partition_sizes:
partition = []
for _ in range(size):
if layer_index < len(model):
partition.append(model[layer_index])
layer_index += 1
partitions.append(nn.Sequential(*partition))
return partitions
model = nn.Sequential(
nn.Conv2d(3, 64, 3),
nn.ReLU(),
nn.Conv2d(64, 128, 3),
nn.ReLU(),
nn.Conv2d(128, 256, 3),
nn.ReLU(),
nn.Linear(256, 1000),
nn.Linear(1000, 10)
)
partition_sizes = [2, 3, 3] # Non-uniform partition sizes
non_uniform_partitions = create_non_uniform_partitions(model, partition_sizes)
gpipe_model = GPipeModule(non_uniform_partitions)
Synchronization and Consistency
GPipe ensures synchronization between different pipeline stages to maintain training consistency. This involves careful management of batch normalization statistics and other stateful operations.
class SynchronizedBatchNorm(nn.BatchNorm2d):
def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True):
super().__init__(num_features, eps, momentum, affine)
self.share_memory()
def forward(self, input):
self._check_input_dim(input)
# Calculate local stats
mean = input.mean([0, 2, 3])
var = input.var([0, 2, 3], unbiased=False)
# Synchronize stats across devices
dist.all_reduce(mean)
dist.all_reduce(var)
mean /= dist.get_world_size()
var /= dist.get_world_size()
return nn.functional.batch_norm(
input, mean, var, self.weight, self.bias,
self.training, self.momentum, self.eps
)
# Replace regular BatchNorm with SynchronizedBatchNorm
for module in model.modules():
if isinstance(module, nn.BatchNorm2d):
module.__class__ = SynchronizedBatchNorm
Handling Variable-Length Sequences
GPipe can be adapted to handle variable-length sequences, which is crucial for natural language processing tasks. This requires careful management of padding and masking within the pipeline.
class VariableLengthGPipeModule(nn.Module):
def __init__(self, partitions):
super().__init__()
self.partitions = nn.ModuleList(partitions)
def forward(self, x, lengths):
max_len = x.size(1)
mask = torch.arange(max_len).expand(len(lengths), max_len) < lengths.unsqueeze(1)
for partition in self.partitions:
x = partition(x)
x = x * mask.unsqueeze(-1).float() # Apply mask after each partition
return x
def backward(self, grad_output, lengths):
max_len = grad_output.size(1)
mask = torch.arange(max_len).expand(len(lengths), max_len) < lengths.unsqueeze(1)
for partition in reversed(self.partitions):
grad_output = partition.backward(grad_output)
grad_output = grad_output * mask.unsqueeze(-1).float()
return grad_output
# Example usage
sentences = ["Hello world", "This is a test", "GPipe is awesome"]
lengths = torch.tensor([len(s.split()) for s in sentences])
input_ids = torch.randint(0, 1000, (len(sentences), max(lengths)))
model = VariableLengthGPipeModule(partitioned_model)
output = model(input_ids, lengths)
Checkpointing and Resuming Training
GPipe supports checkpointing and resuming training, which is essential for long-running experiments and fault tolerance. This involves saving and loading the state of both the model and the optimizer.
import os
def save_checkpoint(model, optimizer, epoch, path):
checkpoint = {
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'epoch': epoch
}
torch.save(checkpoint, path)
def load_checkpoint(model, optimizer, path):
checkpoint = torch.load(path)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
return epoch
# Example usage
model = GPipeModule(partitioned_model)
optimizer = torch.optim.Adam(model.parameters())
# Saving checkpoint
save_checkpoint(model, optimizer, epoch, 'checkpoint.pth')
# Loading checkpoint
if os.path.exists('checkpoint.pth'):
start_epoch = load_checkpoint(model, optimizer, 'checkpoint.pth')
else:
start_epoch = 0
# Resume training
for epoch in range(start_epoch, num_epochs):
train_epoch(model, optimizer, dataloader)
save_checkpoint(model, optimizer, epoch, f'checkpoint_epoch_{epoch}.pth')
Real-Life Example: Language Model Training
GPipe can be used to train large language models that wouldn't fit on a single GPU. Here's an example of how to set up a GPipe-based transformer model for language modeling.
import torch.nn as nn
from GPipe import GPipe
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, num_heads):
super().__init__()
self.attention = nn.MultiheadAttention(embed_dim, num_heads)
self.norm1 = nn.LayerNorm(embed_dim)
self.ffn = nn.Sequential(
nn.Linear(embed_dim, 4 * embed_dim),
nn.ReLU(),
nn.Linear(4 * embed_dim, embed_dim)
)
self.norm2 = nn.LayerNorm(embed_dim)
def forward(self, x):
attn_output, _ = self.attention(x, x, x)
x = self.norm1(x + attn_output)
ffn_output = self.ffn(x)
x = self.norm2(x + ffn_output)
return x
class GPipeTransformer(nn.Module):
def __init__(self, vocab_size, embed_dim, num_heads, num_layers):
super().__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim)
self.transformer_blocks = nn.Sequential(
*[TransformerBlock(embed_dim, num_heads) for _ in range(num_layers)]
)
self.fc = nn.Linear(embed_dim, vocab_size)
def forward(self, x):
x = self.embedding(x)
x = self.transformer_blocks(x)
x = self.fc(x)
return x
# Create a large transformer model
vocab_size = 50000
embed_dim = 1024
num_heads = 16
num_layers = 24
model = GPipeTransformer(vocab_size, embed_dim, num_heads, num_layers)
# Apply GPipe
num_devices = 4
balance = [6] * num_devices # Distribute layers evenly across devices
model = GPipe(model, balance=balance, chunks=8)
# Training loop (simplified)
optimizer = torch.optim.Adam(model.parameters())
criterion = nn.CrossEntropyLoss()
for batch in dataloader:
optimizer.zero_grad()
input_ids, labels = batch
outputs = model(input_ids)
loss = criterion(outputs.view(-1, vocab_size), labels.view(-1))
loss.backward()
optimizer.step()
Real-Life Example: Image Segmentation
GPipe can be applied to large image segmentation models, enabling the training of high-resolution models that exceed single-GPU memory limits. Here's an example using a U-Net architecture.
import torch.nn as nn
from GPipe import GPipe
class DoubleConv(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.double_conv = nn.Sequential(
nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True),
nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True)
)
def forward(self, x):
return self.double_conv(x)
class GPipeUNet(nn.Module):
def __init__(self, n_channels, n_classes):
super().__init__()
self.inc = DoubleConv(n_channels, 64)
self.down1 = nn.Sequential(nn.MaxPool2d(2), DoubleConv(64, 128))
self.down2 = nn.Sequential(nn.MaxPool2d(2), DoubleConv(128, 256))
self.down3 = nn.Sequential(nn.MaxPool2d(2), DoubleConv(256, 512))
self.down4 = nn.Sequential(nn.MaxPool2d(2), DoubleConv(512, 1024))
self.up1 = nn.ConvTranspose2d(1024, 512, kernel_size=2, stride=2)
self.up2 = nn.ConvTranspose2d(512, 256, kernel_size=2, stride=2)
self.up3 = nn.ConvTranspose2d(256, 128, kernel_size=2, stride=2)
self.up4 = nn.ConvTranspose2d(128, 64, kernel_size=2, stride=2)
self.outc = nn.Conv2d(64, n_classes, kernel_size=1)
def forward(self, x):
x1 = self.inc(x)
x2 = self.down1(x1)
x3 = self.down2(x2)
x4 = self.down3(x3)
x5 = self.down4(x4)
x = self.up1(x5) + x4
x = self.up2(x) + x3
x = self.up3(x) + x2
x = self.up4(x) + x1
return self.outc(x)
# Create and partition the model
model = GPipeUNet(n_channels=3, n_classes=10)
model = GPipe(model, balance=[2, 2, 2, 2], chunks=8)
# Training loop (simplified)
optimizer = torch.optim.Adam(model.parameters())
criterion = nn.CrossEntropyLoss()
for batch in dataloader:
optimizer.zero_grad()
images, masks = batch
outputs = model(images)
loss = criterion(outputs, masks)
loss.backward()
optimizer.step()
Handling Dynamic Computation Graphs
GPipe can be adapted to handle models with dynamic computation graphs, such as those found in natural language processing tasks with varying sequence lengths or graph neural networks with different graph structures per batch.
class DynamicGPipeModule(nn.Module):
def __init__(self, partitions):
super().__init__()
self.partitions = nn.ModuleList(partitions)
def forward(self, x, additional_info):
for partition in self.partitions:
x = partition(x, additional_info)
return x
class DynamicPartition(nn.Module):
def __init__(self, static_layer, dynamic_layer):
super().__init__()
self.static_layer = static_layer
self.dynamic_layer = dynamic_layer
def forward(self, x, additional_info):
x = self.static_layer(x)
x = self.dynamic_layer(x, additional_info)
return x
# Example usage
static_layer = nn.Linear(100, 100)
dynamic_layer = lambda x, info: x * info['scale'] + info['bias']
partition = DynamicPartition(static_layer, dynamic_layer)
model = DynamicGPipeModule([partition for _ in range(4)])
x = torch.randn(32, 100)
additional_info = {'scale': torch.randn(32, 100), 'bias': torch.randn(32, 100)}
output = model(x, additional_info)
Performance Optimization Techniques
GPipe can be further optimized using various techniques to improve training speed and efficiency. These include mixed-precision training, gradient accumulation, and adaptive load balancing.
import torch.cuda.amp as amp
class OptimizedGPipeModule(nn.Module):
def __init__(self, partitions, accumulation_steps=4):
super().__init__()
self.partitions = nn.ModuleList(partitions)
self.accumulation_steps = accumulation_steps
self.scaler = amp.GradScaler()
def forward(self, x):
with amp.autocast():
for partition in self.partitions:
x = partition(x)
return x
def backward(self, loss):
self.scaler.scale(loss).backward()
if self.accumulation_steps % self.accumulation_steps == 0:
self.scaler.step(optimizer)
self.scaler.update()
optimizer.zero_grad()
# Adaptive load balancing
def adaptive_balance(model, sample_batch):
with torch.no_grad():
times = []
for partition in model.partitions:
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
_ = partition(sample_batch)
end.record()
torch.cuda.synchronize()
times.append(start.elapsed_time(end))
total_time = sum(times)
new_balance = [int(t / total_time * len(model.partitions)) for t in times]
return new_balance
# Usage
model = OptimizedGPipeModule(partitions)
sample_batch = next(iter(dataloader))
new_balance = adaptive_balance(model, sample_batch)
model = GPipe(model, balance=new_balance, chunks=8)
Additional Resources
For more information on GPipe and pipeline parallelism, refer to the following resources:
- "GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism" (arXiv:1811.06965) - The original GPipe paper. URL: https://arxiv.org/abs/1811.06965
- "PipeDream: Generalized Pipeline Parallelism for DNN Training" (arXiv:1806.03377) - A related approach to pipeline parallelism. URL: https://arxiv.org/abs/1806.03377
- "Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism" (arXiv:1909.08053) - Another approach to training large models. URL: https://arxiv.org/abs/1909.08053
These papers provide in-depth discussions of the techniques and principles behind GPipe and related approaches to training large neural networks.