from pprint import pprint

import numpy as np, pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import StratifiedKFold, GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, balanced_accuracy_score, precision_score, recall_score, f1_score
from scipy.stats import shapiro, ttest_rel, wilcoxon

X, y = make_classification(
    n_samples=2000, n_features=20, n_informative=4, n_redundant=2, n_repeated=0,
    n_clusters_per_class=2, weights=[0.97, 0.03], flip_y=0.0, class_sep=1.0,
    random_state=42
)

outer = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
inner = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
param_grid = {"svc__C": [0.3,1,3,10], 
              "svc__gamma": ["scale", 0.03, 0.1]}
metrics = ["accuracy","balanced_accuracy","precision","recall","f1"]
svm_scores = {m:[] for m in metrics}
knn_scores = {m:[] for m in metrics}
p_value_T = 0.05

for tr, te in outer.split(X,y):
    Xtr, Xte, ytr, yte = X[tr], X[te], y[tr], y[te]

    svm = make_pipeline(StandardScaler(), SVC(kernel="rbf", random_state=42))
    gs = GridSearchCV(svm, param_grid, cv=inner, scoring="balanced_accuracy", n_jobs=1)
    gs.fit(Xtr, ytr)
    yhat_svm = gs.predict(Xte)

    knn = KNeighborsClassifier(n_neighbors=1).fit(Xtr, ytr)
    yhat_knn = knn.predict(Xte)

    for name, func in [
        ("accuracy", accuracy_score),
        ("balanced_accuracy", balanced_accuracy_score),
        ("precision", lambda yt, yp: precision_score(yt, yp, zero_division=0)),
        ("recall",    lambda yt, yp: recall_score(yt, yp, zero_division=0)),
        ("f1",        lambda yt, yp: f1_score(yt, yp, zero_division=0)),
    ]:
        svm_scores[name].append(func(yte, yhat_svm))
        knn_scores[name].append(func(yte, yhat_knn))
pprint(svm_scores)

{'accuracy': [0.975, 0.975, 0.975, 0.96, 0.975, 0.985, 0.97, 0.97, 0.98, 0.965],
 'balanced_accuracy': [0.5833333333333334,
                       0.6640893470790378,
                       0.5833333333333334,
                       0.4948453608247423,
                       0.6640893470790378,
                       0.75,
                       0.5807560137457045,
                       0.5,
                       0.7474226804123711,
                       0.6589347079037801],
 'f1': [0.2857142857142857,
        0.4444444444444444,
        0.2857142857142857,
        0.0,
        0.4444444444444444,
        0.6666666666666666,
        0.25,
        0.0,
        0.6,
        0.36363636363636365],
 'precision': [1.0,
               0.6666666666666666,
               1.0,
               0.0,
               0.6666666666666666,
               1.0,
               0.5,
               0.0,
               0.75,
               0.4],
 'recall': [0.16666666666666666,
            0.3333333333333333,
            0.16666666666666666,
            0.0,
            0.3333333333333333,
            0.5,
            0.16666666666666666,
            0.0,
            0.5,
            0.3333333333333333]}

rows, tests = [], []
for m in metrics:
    s = np.array(svm_scores[m])
    k = np.array(knn_scores[m])
    d = s - k
    W, p_norm = shapiro(d)
    if p_norm >= p_value_T:
        stat, p = ttest_rel(s, k)
        test_used = "paired t-test"
    else:
        stat, p = wilcoxon(s, k)
        test_used = "Wilcoxon signed-rank"
    rows.append([m, s.mean(), s.std(ddof=1), k.mean(), k.std(ddof=1)])
    tests.append([m, test_used, float(p_norm), float(stat), float(p), "YES" if p < p_value_T else "NO"])

summary = pd.DataFrame(rows, columns=["metric","svm_mean","svm_sd","knn1_mean","knn1_sd"]).round(4)
tests_df = pd.DataFrame(tests, columns=["metric","test","shapiro_p","stat","p_value",f"significant(p<{p_value_T:.2f})"]).round(4)
print(summary)
print(tests_df)

              metric  svm_mean  svm_sd  knn1_mean  knn1_sd
0           accuracy    0.9730  0.0071     0.9640   0.0066
1  balanced_accuracy    0.6227  0.0899     0.5777   0.0865
2          precision    0.5983  0.3773     0.2267   0.2130
3             recall    0.2500  0.1800     0.1667   0.1757
4                 f1    0.3341  0.2215     0.1890   0.1876
              metric                  test  shapiro_p    stat  p_value  \
0           accuracy         paired t-test     0.4788  5.0138   0.0007   
1  balanced_accuracy  Wilcoxon signed-rank     0.0036  0.0000   0.0020   
2          precision  Wilcoxon signed-rank     0.0366  0.0000   0.0116   
3             recall  Wilcoxon signed-rank     0.0021  0.0000   0.0588   
4                 f1         paired t-test     0.1019  3.1288   0.0121   

  significant(p<0.05)  
0                 YES  
1                 YES  
2                 YES  
3                  NO  
4                 YES

Ocena wydajności klasyfikatorów¶