Breast Cancer Classification

Imports

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_validate
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import (
    confusion_matrix,
    ConfusionMatrixDisplay,
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    roc_auc_score,
    RocCurveDisplay
)

pd.set_option("display.max_columns", None)

Load Dataset

data = load_breast_cancer(as_frame=True)

X = data.data.copy()

# 1 = malignant, 0 = benign
y = (data.target == 0).astype(int)

print("Dataset shape:", X.shape)
print("\nClass counts:")
print(y.value_counts().rename(index={0: "Benign", 1: "Malignant"}))

Dataset shape: (569, 30)

Class counts:
target
Benign       357
Malignant    212
Name: count, dtype: int64

Remove 105 Observations

AGE = 35
N_REMOVE = 3 * AGE
RANDOM_SEED = 42

rng = np.random.default_rng(RANDOM_SEED)

remove_indices = rng.choice(
    X.index,
    size=N_REMOVE,
    replace=False
)

X_reduced = X.drop(index=remove_indices).reset_index(drop=True)
y_reduced = y.drop(index=remove_indices).reset_index(drop=True)

print("Removed:", N_REMOVE)
print("Remaining observations:", len(X_reduced))

Removed: 105
Remaining observations: 464

Class Distribution

distribution = pd.DataFrame({
    "Count": y_reduced.value_counts().sort_index(),
    "Percentage": y_reduced.value_counts(normalize=True).sort_index() * 100
})

distribution.index = ["Benign", "Malignant"]
distribution

	Count	Percentage
Benign	290	62.5
Malignant	174	37.5

Train-Test Split

X_train, X_test, y_train, y_test = train_test_split(
    X_reduced,
    y_reduced,
    test_size=0.20,
    random_state=RANDOM_SEED,
    stratify=y_reduced
)

print("Training set:", len(X_train))
print("Test set:", len(X_test))

Training set: 371
Test set: 93

Train Models

logistic_model = Pipeline([
    ("scaler", StandardScaler()),
    ("classifier", LogisticRegression(
        max_iter=5000,
        random_state=RANDOM_SEED
    ))
])

tree_model = DecisionTreeClassifier(
    max_depth=4,
    random_state=RANDOM_SEED
)

rf_model = RandomForestClassifier(
    n_estimators=300,
    random_state=RANDOM_SEED,
    n_jobs=-1
)

logistic_model.fit(X_train, y_train)
tree_model.fit(X_train, y_train)
rf_model.fit(X_train, y_train)

print("All models trained.")

All models trained.

Evaluation Function

def evaluate_model(name, model):
    y_pred = model.predict(X_test)
    y_prob = model.predict_proba(X_test)[:, 1]

    cm = confusion_matrix(y_test, y_pred, labels=[0, 1])
    tn, fp, fn, tp = cm.ravel()

    result = {
        "Model": name,
        "Accuracy": accuracy_score(y_test, y_pred),
        "Precision": precision_score(y_test, y_pred, zero_division=0),
        "Sensitivity": recall_score(y_test, y_pred, zero_division=0),
        "Specificity": tn / (tn + fp),
        "F1 Score": f1_score(y_test, y_pred, zero_division=0),
        "ROC-AUC": roc_auc_score(y_test, y_prob),
        "TN": tn,
        "FP": fp,
        "FN": fn,
        "TP": tp
    }

    print(name)
    print(result)

    ConfusionMatrixDisplay(
        confusion_matrix=cm,
        display_labels=["Benign", "Malignant"]
    ).plot(values_format="d")

    plt.title(f"{name} Confusion Matrix")
    plt.show()

    return result

Logistic Regression

logistic_result = evaluate_model(
    "Logistic Regression",
    logistic_model
)

Logistic Regression
{'Model': 'Logistic Regression', 'Accuracy': 0.989247311827957, 'Precision': 1.0, 'Sensitivity': 0.9714285714285714, 'Specificity': np.float64(1.0), 'F1 Score': 0.9855072463768116, 'ROC-AUC': 0.9911330049261085, 'TN': np.int64(58), 'FP': np.int64(0), 'FN': np.int64(1), 'TP': np.int64(34)}

Decision Tree

tree_result = evaluate_model(
    "Decision Tree",
    tree_model
)

Decision Tree
{'Model': 'Decision Tree', 'Accuracy': 0.9247311827956989, 'Precision': 0.9117647058823529, 'Sensitivity': 0.8857142857142857, 'Specificity': np.float64(0.9482758620689655), 'F1 Score': 0.8985507246376812, 'ROC-AUC': 0.9312807881773398, 'TN': np.int64(55), 'FP': np.int64(3), 'FN': np.int64(4), 'TP': np.int64(31)}

Random Forest

rf_result = evaluate_model(
    "Random Forest",
    rf_model
)

Random Forest
{'Model': 'Random Forest', 'Accuracy': 0.9247311827956989, 'Precision': 0.8888888888888888, 'Sensitivity': 0.9142857142857143, 'Specificity': np.float64(0.9310344827586207), 'F1 Score': 0.9014084507042254, 'ROC-AUC': 0.9896551724137931, 'TN': np.int64(54), 'FP': np.int64(4), 'FN': np.int64(3), 'TP': np.int64(32)}

Model Comparison

results_df = pd.DataFrame([
    logistic_result,
    tree_result,
    rf_result
])

results_df.round(4)

	Model	Accuracy	Precision	Sensitivity	Specificity	F1 Score	ROC-AUC	TN	FP	FN	TP
0	Logistic Regression	0.9892	1.0000	0.9714	1.0000	0.9855	0.9911	58	0	1	34
1	Decision Tree	0.9247	0.9118	0.8857	0.9483	0.8986	0.9313	55	3	4	31
2	Random Forest	0.9247	0.8889	0.9143	0.9310	0.9014	0.9897	54	4	3	32

ROC Curves

fig, ax = plt.subplots(figsize=(8, 6))

RocCurveDisplay.from_estimator(
    logistic_model,
    X_test,
    y_test,
    name="Logistic Regression",
    ax=ax
)

RocCurveDisplay.from_estimator(
    tree_model,
    X_test,
    y_test,
    name="Decision Tree",
    ax=ax
)

RocCurveDisplay.from_estimator(
    rf_model,
    X_test,
    y_test,
    name="Random Forest",
    ax=ax
)

plt.title("ROC Curves")
plt.show()

5-Fold Cross-Validation

cv = StratifiedKFold(
    n_splits=5,
    shuffle=True,
    random_state=RANDOM_SEED
)

scoring = {
    "accuracy": "accuracy",
    "precision": "precision",
    "sensitivity": "recall",
    "f1": "f1",
    "roc_auc": "roc_auc"
}

models = {
    "Logistic Regression": logistic_model,
    "Decision Tree": tree_model,
    "Random Forest": rf_model
}

cv_results = []

for name, model in models.items():
    scores = cross_validate(
        model,
        X_reduced,
        y_reduced,
        cv=cv,
        scoring=scoring,
        n_jobs=-1
    )

    cv_results.append({
        "Model": name,
        "CV Accuracy": scores["test_accuracy"].mean(),
        "CV Precision": scores["test_precision"].mean(),
        "CV Sensitivity": scores["test_sensitivity"].mean(),
        "CV F1": scores["test_f1"].mean(),
        "CV ROC-AUC": scores["test_roc_auc"].mean()
    })

cv_results_df = pd.DataFrame(cv_results)
cv_results_df.round(4)

	Model	CV Accuracy	CV Precision	CV Sensitivity	CV F1	CV ROC-AUC
0	Logistic Regression	0.9763	0.9828	0.9543	0.9678	0.9955
1	Decision Tree	0.9311	0.9264	0.8911	0.9071	0.9267
2	Random Forest	0.9677	0.9604	0.9543	0.9566	0.9904

Final Summary

summary_df = results_df.merge(
    cv_results_df,
    on="Model"
)

summary_df.round(4)

	Model	Accuracy	Precision	Sensitivity	Specificity	F1 Score	ROC-AUC	TN	FP	FN	TP	CV Accuracy	CV Precision	CV Sensitivity	CV F1	CV ROC-AUC
0	Logistic Regression	0.9892	1.0000	0.9714	1.0000	0.9855	0.9911	58	0	1	34	0.9763	0.9828	0.9543	0.9678	0.9955
1	Decision Tree	0.9247	0.9118	0.8857	0.9483	0.8986	0.9313	55	3	4	31	0.9311	0.9264	0.8911	0.9071	0.9267
2	Random Forest	0.9247	0.8889	0.9143	0.9310	0.9014	0.9897	54	4	3	32	0.9677	0.9604	0.9543	0.9566	0.9904

print("Fewest false negatives:")
print(results_df.loc[results_df["FN"].idxmin(), ["Model", "FN"]])

print("\nHighest sensitivity:")
print(results_df.loc[results_df["Sensitivity"].idxmax(), ["Model", "Sensitivity"]])

print("\nHighest ROC-AUC:")
print(results_df.loc[results_df["ROC-AUC"].idxmax(), ["Model", "ROC-AUC"]])

print("\nHighest CV sensitivity:")
print(cv_results_df.loc[
    cv_results_df["CV Sensitivity"].idxmax(),
    ["Model", "CV Sensitivity"]
])

Fewest false negatives:
Model    Logistic Regression
FN                         1
Name: 0, dtype: object

Highest sensitivity:
Model          Logistic Regression
Sensitivity               0.971429
Name: 0, dtype: object

Highest ROC-AUC:
Model      Logistic Regression
ROC-AUC               0.991133
Name: 0, dtype: object

Highest CV sensitivity:
Model             Logistic Regression
CV Sensitivity               0.954286
Name: 0, dtype: object