wal.sh
home > research > machine-learning-python · Jan 01, 2024

Modern Python Machine Learning Ecosystem Guide (2024-2025)

python machine learningpytorchtensorflowhugging facescikit-learntransformersllmpython 3.11+

Modern Python Machine Learning Ecosystem Guide (2024-2025)

Table of Contents

1. Overview

The Python ML ecosystem has matured significantly with Python 3.11+ offering substantial performance improvements and enhanced type hints. This guide covers the essential tools and workflows for modern machine learning development.

2. Core Python Environment

2.1. Python 3.11+ Features

  • Performance: 10-60% faster than 3.10
  • Better error messages with precise line numbers
  • Enhanced type annotations and typing.Self
  • Task groups for async workflows

2.2. Environment Management

# Using uv (modern fast package manager)
curl -LsSf https://astral.sh/uv/install.sh | sh
uv venv
source .venv/bin/activate
uv pip install torch torchvision scikit-learn pandas

3. Essential ML Libraries

3.1. Core Frameworks

3.1.1. PyTorch (Recommended for Research & Production)

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset

class SimpleModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super().__init__()
        self.layers = nn.Sequential(
            nn.Linear(input_dim, hidden_dim),
            nn.ReLU(),
            nn.Dropout(0.2),
            nn.Linear(hidden_dim, output_dim)
        )

    def forward(self, x):
        return self.layers(x)

# Training loop
model = SimpleModel(784, 128, 10)
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()

3.1.2. TensorFlow/Keras (Production & Deployment)

import tensorflow as tf
from tensorflow import keras

model = keras.Sequential([
    keras.layers.Dense(128, activation='relu', input_shape=(784,)),
    keras.layers.Dropout(0.2),
    keras.layers.Dense(10, activation='softmax')
])

model.compile(
    optimizer=keras.optimizers.AdamW(learning_rate=1e-3),
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

3.2. Data Processing Libraries

3.2.1. Pandas & Polars

import pandas as pd
import polars as pl

# Pandas: mature, comprehensive
df_pandas = pd.read_csv('data.csv')
df_pandas['feature'] = df_pandas['col1'] * df_pandas['col2']

# Polars: faster for large datasets
df_polars = pl.read_csv('data.csv')
df_polars = df_polars.with_columns(
    (pl.col('col1') * pl.col('col2')).alias('feature')
)

3.2.2. NumPy

import numpy as np

# Vectorized operations
X = np.random.randn(1000, 100)
X_normalized = (X - X.mean(axis=0)) / X.std(axis=0)

3.3. Traditional ML: Scikit-learn

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

# Complete ML pipeline
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('classifier', RandomForestClassifier(n_estimators=100, random_state=42))
])

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
pipeline.fit(X_train, y_train)
scores = cross_val_score(pipeline, X_train, y_train, cv=5)
print(f"CV Accuracy: {scores.mean():.3f} (+/- {scores.std():.3f})")

4. Modern ML Workflow

4.1. Data Preparation

from sklearn.model_selection import train_test_split
from torch.utils.data import Dataset, DataLoader

class CustomDataset(Dataset):
    def __init__(self, features, labels):
        self.features = torch.FloatTensor(features)
        self.labels = torch.LongTensor(labels)

    def __len__(self):
        return len(self.features)

    def __getitem__(self, idx):
        return self.features[idx], self.labels[idx]

train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)

4.2. Training with Monitoring

from tqdm import tqdm

def train_epoch(model, loader, optimizer, criterion, device):
    model.train()
    total_loss = 0

    for batch_x, batch_y in tqdm(loader, desc="Training"):
        batch_x, batch_y = batch_x.to(device), batch_y.to(device)

        optimizer.zero_grad()
        outputs = model(batch_x)
        loss = criterion(outputs, batch_y)
        loss.backward()
        optimizer.step()

        total_loss += loss.item()

    return total_loss / len(loader)

4.3. Evaluation & Metrics

from sklearn.metrics import classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

def evaluate_model(model, loader, device):
    model.eval()
    predictions, actuals = [], []

    with torch.no_grad():
        for batch_x, batch_y in loader:
            batch_x = batch_x.to(device)
            outputs = model(batch_x)
            preds = outputs.argmax(dim=1).cpu().numpy()
            predictions.extend(preds)
            actuals.extend(batch_y.numpy())

    print(classification_report(actuals, predictions))
    return predictions, actuals

5. LLM & Transformers Integration

5.1. Hugging Face Transformers

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    Trainer,
    TrainingArguments
)

# Load pretrained model
model_name = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(
    model_name,
    num_labels=2
)

# Tokenize data
def tokenize_function(examples):
    return tokenizer(
        examples["text"],
        padding="max_length",
        truncation=True,
        max_length=512
    )

# Training
training_args = TrainingArguments(
    output_dir="./results",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    num_train_epochs=3,
    logging_dir="./logs",
    evaluation_strategy="epoch"
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset
)

trainer.train()

5.2. Quick Inference with Pipeline

from transformers import pipeline

# Text classification
classifier = pipeline("text-classification", model="distilbert-base-uncased")
result = classifier("This is a great product!")

# Text generation
generator = pipeline("text-generation", model="gpt2")
output = generator("Machine learning is", max_length=50)

# Question answering
qa = pipeline("question-answering")
answer = qa(question="What is ML?", context="Machine learning is...")

6. Advanced Topics

6.1. Model Optimization

# Quantization for faster inference
import torch.quantization as quantization

model.eval()
model_quantized = quantization.quantize_dynamic(
    model, {nn.Linear}, dtype=torch.qint8
)

# Mixed precision training
from torch.cuda.amp import GradScaler, autocast

scaler = GradScaler()
with autocast():
    outputs = model(inputs)
    loss = criterion(outputs, targets)

scaler.scale(loss).backward()
scaler.step(optimizer)
scaler.update()

6.2. Experiment Tracking

import wandb

wandb.init(project="my-ml-project", config={"lr": 1e-3, "epochs": 10})

for epoch in range(epochs):
    train_loss = train_epoch(model, train_loader, optimizer, criterion)
    val_loss = validate(model, val_loader, criterion)

    wandb.log({"train_loss": train_loss, "val_loss": val_loss})

7. Essential Resources

7.2. Learning Paths

7.3. Tools & Platforms

  • Jupyter Lab: Interactive development
  • VS Code + Python extension: Full IDE experience
  • Google Colab: Free GPU access
  • Weights & Biases: Experiment tracking
  • Gradio/Streamlit: Quick ML app deployment

8. Best Practices

  • Use type hints for better code quality: def train(model: nn.Module, data: DataLoader) -> float
  • Implement reproducibility: Set random seeds (torch.manual_seed(42))
  • Version control data and models: DVC, Git LFS
  • Monitor GPU memory: torch.cuda.memory_summary()
  • Use virtual environments: venv, conda, or uv
  • Profile code: torch.profiler, cProfile
  • Validate inputs and outputs with assertions
  • Document model architectures and hyperparameters