Data Preprocessing

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.model_selection import train_test_split

RANDOM_STATE = 42
rng = np.random.default_rng(RANDOM_STATE)

pd.set_option("display.width", 110)
sns.set_theme(style="whitegrid")

customers = pd.read_csv("data/customers.csv").drop_duplicates()
print(customers.shape)

(1000, 14)

train, test = train_test_split(
    customers, test_size=0.2, stratify=customers["churned"], random_state=RANDOM_STATE
)
print(f"train {train.shape}   test {test.shape}")
print(f"churn - train {train['churned'].mean():.3f}  test {test['churned'].mean():.3f}")

train (800, 14)   test (200, 14)
churn - train 0.141  test 0.140

rates_stratified, rates_plain = [], []
for seed in range(200):
    _, t_strat = train_test_split(customers, test_size=0.2, stratify=customers["churned"],
                                  random_state=seed)
    _, t_plain = train_test_split(customers, test_size=0.2, random_state=seed)
    rates_stratified.append(t_strat["churned"].mean())
    rates_plain.append(t_plain["churned"].mean())

print(f"test churn rate over 200 splits, stratified: "
      f"{np.min(rates_stratified):.3f} - {np.max(rates_stratified):.3f}")
print(f"test churn rate over 200 splits, plain     : "
      f"{np.min(rates_plain):.3f} - {np.max(rates_plain):.3f}")

test churn rate over 200 splits, stratified: 0.140 - 0.140
test churn rate over 200 splits, plain     : 0.085 - 0.205

from sklearn.preprocessing import StandardScaler

col = customers[["monthly_charges"]].dropna()
tr, te = train_test_split(col, test_size=0.2, random_state=RANDOM_STATE)

leaky = StandardScaler().fit(col)   # WRONG: fitted on everything, test rows included
clean = StandardScaler().fit(tr)     # RIGHT: fitted on training rows only

print(f"mean used, leaky : {leaky.mean_[0]:.6f}")
print(f"mean used, clean : {clean.mean_[0]:.6f}")
print(f"difference       : {abs(leaky.mean_[0] - clean.mean_[0]):.6f}")

mean used, leaky : 40.384799
mean used, clean : 40.524394
difference       : 0.139595

missing_flag = train["satisfaction_score"].isna()

comparison = pd.DataFrame({
    "satisfaction present": train.loc[~missing_flag, ["tenure_months", "support_tickets", "churned"]].mean(),
    "satisfaction missing": train.loc[missing_flag, ["tenure_months", "support_tickets", "churned"]].mean(),
}).round(3)
print(comparison)
print(f"\nrows with satisfaction missing: {missing_flag.sum()} of {len(train)}")

                 satisfaction present  satisfaction missing
tenure_months                  36.116                38.649
support_tickets                 1.244                 1.351
churned                         0.147                 0.081

rows with satisfaction missing: 74 of 800

present = train.loc[~missing_flag, "churned"]
absent = train.loc[missing_flag, "churned"]

def se(s):
    p = s.mean()
    return np.sqrt(p * (1 - p) / len(s))

diff = present.mean() - absent.mean()
se_diff = np.sqrt(se(present) ** 2 + se(absent) ** 2)

print(f"churn | present : {present.mean():.3f}  (n={len(present)})")
print(f"churn | missing : {absent.mean():.3f}  (n={len(absent)})")
print(f"difference      : {diff:.3f}  +/- {se_diff:.3f} (1 s.e.)  -> {diff / se_diff:.1f} standard errors")

churn | present : 0.147  (n=726)
churn | missing : 0.081  (n=74)
difference      : 0.066  +/- 0.034 (1 s.e.)  -> 1.9 standard errors

n = len(train)
print(f"rows: {n}")
for col in ["age", "gender", "satisfaction_score", "monthly_charges"]:
    print(f"  {col:<20} missing {train[col].isna().sum():>3}  ({train[col].isna().mean():.1%})")

complete = train.drop(columns=["cancellation_reason"]).dropna()
print(f"\nrows surviving a listwise dropna(): {len(complete)}  ({len(complete) / n:.1%})")
print(f"rows lost: {n - len(complete)}")

rows: 800
  age                  missing  46  (5.8%)
  gender               missing  36  (4.5%)
  satisfaction_score   missing  74  (9.2%)
  monthly_charges      missing  24  (3.0%)

rows surviving a listwise dropna(): 638  (79.8%)
rows lost: 162

from sklearn.impute import SimpleImputer

num_cols = ["age", "monthly_charges", "satisfaction_score"]

median_imp = SimpleImputer(strategy="median")
median_imp.fit(train[num_cols])       # LEARN the medians - from train only (5.1)

train_filled = pd.DataFrame(median_imp.transform(train[num_cols]),
                            columns=num_cols, index=train.index)
test_filled = pd.DataFrame(median_imp.transform(test[num_cols]),
                           columns=num_cols, index=test.index)

print("medians learned:", {c: float(v) for c, v in zip(num_cols, median_imp.statistics_.round(2))})
print("nulls after, train:", train_filled.isna().sum().sum(), "| test:", test_filled.isna().sum().sum())

medians learned: {'age': 38.0, 'monthly_charges': 38.47, 'satisfaction_score': 7.0}
nulls after, train: 0 | test: 0

raw_median = train["age"].median()

age_cleaned = train["age"].where(train["age"].between(16, 100))   # impossible -> NaN
clean_median = age_cleaned.median()

print(f"median age including impossible values: {raw_median:.1f}")
print(f"median age after blanking them        : {clean_median:.1f}")
print(f"mean   age including impossible values: {train['age'].mean():.2f}")
print(f"mean   age after blanking them        : {age_cleaned.mean():.2f}")

median age including impossible values: 38.0
median age after blanking them        : 38.0
mean   age including impossible values: 38.16
mean   age after blanking them        : 37.89

indicator_imp = SimpleImputer(strategy="median", add_indicator=True)
result = indicator_imp.fit_transform(train[num_cols])

print("input columns :", len(num_cols))
print("output columns:", result.shape[1], "-> 3 imputed + 3 boolean 'was missing' flags")
print("\nis missingness related to churn?")
for col in num_cols:
    flag = train[col].isna()
    if flag.any():
        print(f"  {col:<20} churn when present {train.loc[~flag, 'churned'].mean():.3f}"
              f"   when missing {train.loc[flag, 'churned'].mean():.3f}")

input columns : 3
output columns: 6 -> 3 imputed + 3 boolean 'was missing' flags

is missingness related to churn?
  age                  churn when present 0.139   when missing 0.174
  monthly_charges      churn when present 0.143   when missing 0.083
  satisfaction_score   churn when present 0.147   when missing 0.081

def iqr_bounds(s, k=1.5):
    q1, q3 = s.quantile([0.25, 0.75])
    iqr = q3 - q1
    return q1 - k * iqr, q3 + k * iqr


for col in ["age", "monthly_charges", "support_tickets", "last_login_days"]:
    s = train[col].dropna()
    lo, hi = iqr_bounds(s)
    z = np.abs((s - s.mean()) / s.std())
    print(f"{col:<18} IQR flags {((s < lo) | (s > hi)).sum():>3}   z>3 flags {(z > 3).sum():>3}"
          f"   bounds [{lo:.1f}, {hi:.1f}]")

age                IQR flags   9   z>3 flags   2   bounds [7.5, 67.5]
monthly_charges    IQR flags   0   z>3 flags   0   bounds [-14.2, 93.3]
support_tickets    IQR flags   1   z>3 flags   6   bounds [-3.0, 5.0]
last_login_days    IQR flags  25   z>3 flags  10   bounds [-8.6, 32.4]

tiny = np.array([48.0, 51.0, 49.0, 52.0, 50.0, 47.0, 53.0, 49.0, 51.0, 1_000_000.0])
n = len(tiny)

z = np.abs((tiny - tiny.mean()) / tiny.std(ddof=1))   # ddof=1: the sample standard deviation
lo, hi = iqr_bounds(pd.Series(tiny))

print("one value is a million; the other nine are around 50")
print(f"its z-score        : {z.max():.3f}")
print(f"theoretical maximum: {(n - 1) / np.sqrt(n):.3f}   <- z can never exceed this, at any magnitude")
print(f"z > 3 catches      : {(z > 3).sum()}  <- nothing, and it never could")
print(f"IQR rule catches   : {((tiny < lo) | (tiny > hi)).sum()}")

one value is a million; the other nine are around 50
its z-score        : 2.846
theoretical maximum: 2.846   <- z can never exceed this, at any magnitude
z > 3 catches      : 0  <- nothing, and it never could
IQR rule catches   : 1

# Impossible -> NaN. Note this is fitted knowledge-free: the bounds come from the DOMAIN, not the data,
# so applying the same rule to train and test introduces no leakage.
DOMAIN_RULES = {"age": (16, 100), "satisfaction_score": (1, 10), "monthly_charges": (0, 500)}

def apply_domain_rules(df):
    out = df.copy()
    for col, (lo, hi) in DOMAIN_RULES.items():
        out[col] = out[col].where(out[col].between(lo, hi))
    return out


train_repaired = apply_domain_rules(train)
test_repaired = apply_domain_rules(test)

print("age nulls  before:", train["age"].isna().sum(), " after:", train_repaired["age"].isna().sum())
print("age range  after :", train_repaired["age"].min(), "-", train_repaired["age"].max())

age nulls  before: 46  after: 50
age range  after : 18.0 - 76.0

# Winsorizing, when it IS wanted: bounds learned from TRAIN, applied to both.
lo_p, hi_p = train_repaired["monthly_charges"].quantile([0.01, 0.99])

train_clipped = train_repaired["monthly_charges"].clip(lo_p, hi_p)
test_clipped = test_repaired["monthly_charges"].clip(lo_p, hi_p)   # train's bounds, not test's

print(f"clip bounds from TRAIN: [{lo_p:.2f}, {hi_p:.2f}]")
print(f"train range after: [{train_clipped.min():.2f}, {train_clipped.max():.2f}]")
print(f"test  range after: [{test_clipped.min():.2f}, {test_clipped.max():.2f}]")

clip bounds from TRAIN: [15.00, 78.50]
train range after: [15.00, 78.50]
test  range after: [15.00, 78.50]

raw = pd.read_csv("data/customers.csv")
print("rows as loaded        :", len(raw))
print("exact duplicate rows  :", raw.duplicated().sum())
print("duplicate customer_id :", raw["customer_id"].duplicated().sum())
print("rows after dedup      :", len(raw.drop_duplicates()))

rows as loaded        : 1012
exact duplicate rows  : 12
duplicate customer_id : 12
rows after dedup      : 1000

print("before:", raw["total_charges"].dtype)

repaired = pd.to_numeric(raw["total_charges"], errors="coerce")
print("after :", repaired.dtype, "| new NaNs:", repaired.isna().sum())

# Those 9 NaNs are not random - they are the brand-new accounts, and that is meaningful.
brand_new = raw.loc[repaired.isna(), "tenure_months"]
print("tenure of the rows that failed to parse:", sorted(int(v) for v in brand_new.unique()))   # [1]

before: str
after : float64 | new NaNs: 9
tenure of the rows that failed to parse: [1]

total_charges_fixed = repaired.where(repaired.notna() | (raw["tenure_months"] > 1), 0.0)
print("nulls remaining:", total_charges_fixed.isna().sum())
print("the 9 repaired rows now read:", total_charges_fixed[repaired.isna()].unique())

nulls remaining: 0
the 9 repaired rows now read: [0.]

def canonical_gender(s):
    return s.str.strip().str.lower().replace({"male": "m", "female": "f"})


print("before:", sorted(raw["gender"].dropna().unique()))
print("after :", sorted(canonical_gender(raw["gender"]).dropna().unique()))

before: ['F', 'Female', 'M', 'Male ', 'female', 'male']
after : ['f', 'm']

from sklearn.preprocessing import MaxAbsScaler, MinMaxScaler, RobustScaler, StandardScaler

demo = train_repaired[["tenure_months", "monthly_charges"]].dropna()

for name, scaler in [("StandardScaler", StandardScaler()),
                     ("MinMaxScaler", MinMaxScaler()),
                     ("RobustScaler", RobustScaler()),
                     ("MaxAbsScaler", MaxAbsScaler())]:
    scaled = scaler.fit_transform(demo)
    print(f"{name:<16} tenure [{scaled[:, 0].min():>6.2f}, {scaled[:, 0].max():>5.2f}]"
          f"   charges [{scaled[:, 1].min():>6.2f}, {scaled[:, 1].max():>5.2f}]")

StandardScaler   tenure [ -1.69,  1.71]   charges [ -1.49,  2.87]
MinMaxScaler     tenure [  0.00,  1.00]   charges [  0.00,  1.00]
RobustScaler     tenure [ -0.92,  1.00]   charges [ -0.87,  1.91]
MaxAbsScaler     tenure [  0.01,  1.00]   charges [  0.17,  1.00]

with_outlier = np.append(rng.normal(50, 5, 200), 5000.0).reshape(-1, 1)

minmax = MinMaxScaler().fit_transform(with_outlier)
robust = RobustScaler().fit_transform(with_outlier)

print("MinMax : 99% of the data now spans "
      f"[{minmax[:-1].min():.4f}, {minmax[:-1].max():.4f}] - squeezed into 1% of the range")
print("Robust : the same 99% spans "
      f"[{robust[:-1].min():.2f}, {robust[:-1].max():.2f}] - the outlier is out at "
      f"{robust[-1, 0]:.0f} where it belongs")

MinMax : 99% of the data now spans [0.0000, 0.0051] - squeezed into 1% of the range
Robust : the same 99% spans [-1.74, 2.47] - the outlier is out at 826 where it belongs

from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import make_pipeline
from sklearn.metrics import roc_auc_score

feat = ["tenure_months", "monthly_charges", "support_tickets", "satisfaction_score"]

tr = train_repaired[feat + ["churned"]].dropna()
te = test_repaired[feat + ["churned"]].dropna()

unscaled = KNeighborsClassifier().fit(tr[feat], tr["churned"])
scaled = make_pipeline(StandardScaler(), KNeighborsClassifier()).fit(tr[feat], tr["churned"])

for name, model in (("kNN, unscaled", unscaled), ("kNN, scaled", scaled)):
    auc = roc_auc_score(te["churned"], model.predict_proba(te[feat])[:, 1])
    print(f"{name:<16} test AUC {auc:.3f}")

kNN, unscaled    test AUC 0.688
kNN, scaled      test AUC 0.720

from sklearn.preprocessing import PowerTransformer, QuantileTransformer

skewed = train_repaired["last_login_days"].dropna().to_numpy().reshape(-1, 1)

variants = {
    "original": skewed.ravel(),
    "log1p": np.log1p(skewed).ravel(),
    "sqrt": np.sqrt(skewed).ravel(),
    "yeo-johnson": PowerTransformer(method="yeo-johnson").fit_transform(skewed).ravel(),
    "quantile-normal": QuantileTransformer(output_distribution="normal", n_quantiles=200,
                                           random_state=RANDOM_STATE).fit_transform(skewed).ravel(),
}

for name, values in variants.items():
    print(f"{name:<18} skew {pd.Series(values).skew():>6.2f}")

original           skew   1.26
log1p              skew  -0.38
sqrt               skew   0.35
yeo-johnson        skew  -0.01
quantile-normal    skew   0.03

fig, axes = plt.subplots(1, 5, figsize=(15, 2.8))
for ax, (name, values) in zip(axes, variants.items()):
    ax.hist(values, bins=30, color="tab:cyan")
    ax.set_title(f"{name}\nskew {pd.Series(values).skew():.2f}", fontsize=9)
    ax.set_yticks([])
fig.suptitle("Same feature, five transforms")
fig.tight_layout()
plt.show()

print("zeros in last_login_days:", (skewed == 0).sum())

try:
    PowerTransformer(method="box-cox").fit_transform(skewed)
except ValueError as e:
    print("box-cox ->", str(e)[:80])

print("yeo-johnson handles it:",
      PowerTransformer(method="yeo-johnson").fit_transform(skewed).shape)

zeros in last_login_days: 1
box-cox -> The Box-Cox transformation can only be applied to strictly positive data
yeo-johnson handles it: (800, 1)

from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score

tree_frame = train_repaired[["last_login_days", "support_tickets", "churned"]].dropna()
X_raw = tree_frame[["last_login_days", "support_tickets"]]
X_log = np.log1p(X_raw)

for name, X_ in (("raw", X_raw), ("log1p", X_log)):
    scores = cross_val_score(DecisionTreeClassifier(max_depth=4, random_state=RANDOM_STATE),
                             X_, tree_frame["churned"], cv=5, scoring="roc_auc")
    print(f"decision tree on {name:<6} mean AUC {scores.mean():.4f}")

decision tree on raw    mean AUC 0.6727
decision tree on log1p  mean AUC 0.6727

def preprocess(df, fitted=None):
    """Steps 2-5: deterministic repairs that learn nothing from the data."""
    out = df.drop_duplicates().copy()
    out["total_charges"] = pd.to_numeric(out["total_charges"], errors="coerce")
    out.loc[out["total_charges"].isna() & (out["tenure_months"] <= 1), "total_charges"] = 0.0
    out["gender"] = canonical_gender(out["gender"])
    for col, (lo, hi) in DOMAIN_RULES.items():
        out[col] = out[col].where(out[col].between(lo, hi))
    return out


train_prepared = preprocess(train)
test_prepared = preprocess(test)

print("train:", train_prepared.shape, " test:", test_prepared.shape)
print("gender levels:", sorted(train_prepared["gender"].dropna().unique()))
print("age range    :", train_prepared["age"].min(), "-", train_prepared["age"].max())
print("total_charges dtype:", train_prepared["total_charges"].dtype)

train: (800, 14)  test: (200, 14)
gender levels: ['f', 'm']
age range    : 18.0 - 76.0
total_charges dtype: float64

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder

numeric_features = ["age", "tenure_months", "monthly_charges", "total_charges",
                    "support_tickets", "last_login_days", "satisfaction_score"]
categorical_features = ["gender", "city", "plan"]

learned_steps = ColumnTransformer([
    ("num", make_pipeline(SimpleImputer(strategy="median", add_indicator=True),
                          StandardScaler()), numeric_features),
    ("cat", make_pipeline(SimpleImputer(strategy="most_frequent"),
                          OneHotEncoder(handle_unknown="ignore")), categorical_features),
])

X_train = learned_steps.fit_transform(train_prepared)    # FIT on train
X_test = learned_steps.transform(test_prepared)           # APPLY to test

print("train matrix:", X_train.shape)
print("test  matrix:", X_test.shape, "-> same width, which is the point")

train matrix: (800, 23)
test  matrix: (200, 23) -> same width, which is the point

from sklearn.linear_model import LogisticRegression

feature_set = ["age", "tenure_months", "monthly_charges", "support_tickets",
               "last_login_days", "satisfaction_score"]

def evaluate(name, train_X, train_y, test_X, test_y, model):
    model.fit(train_X, train_y)
    auc = roc_auc_score(test_y, model.predict_proba(test_X)[:, 1])
    print(f"{name:<38} n_train {len(train_X):>4}   test AUC {auc:.3f}")
    return auc


# A: drop every row with any missing value, no repairs at all.
a_tr = train[feature_set + ["churned"]].dropna()
a_te = test[feature_set + ["churned"]].dropna()
evaluate("A. dropna, no repairs", a_tr[feature_set], a_tr["churned"],
         a_te[feature_set], a_te["churned"],
         make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000)))

A. dropna, no repairs                  n_train  666   test AUC 0.823
0.8232271325796505

# B: mean-impute everything, still no repairs - so age 199 is inside the mean.
b_model = make_pipeline(SimpleImputer(strategy="mean"), StandardScaler(),
                        LogisticRegression(max_iter=2000))
evaluate("B. mean impute, no repairs", train[feature_set], train["churned"],
         test[feature_set], test["churned"], b_model)

# C: repair impossible values first, then median-impute.
c_model = make_pipeline(SimpleImputer(strategy="median"), StandardScaler(),
                        LogisticRegression(max_iter=2000))
evaluate("C. repair, then median impute", train_prepared[feature_set], train_prepared["churned"],
         test_prepared[feature_set], test_prepared["churned"], c_model)

# D: C, plus missingness indicators.
d_model = make_pipeline(SimpleImputer(strategy="median", add_indicator=True), StandardScaler(),
                        LogisticRegression(max_iter=2000))
evaluate("D. repair + median impute + indicators", train_prepared[feature_set],
         train_prepared["churned"], test_prepared[feature_set], test_prepared["churned"], d_model)

B. mean impute, no repairs             n_train  800   test AUC 0.830
C. repair, then median impute          n_train  800   test AUC 0.831
D. repair + median impute + indicators n_train  800   test AUC 0.830
0.8299418604651163