122 lines
4.6 KiB
Python
122 lines
4.6 KiB
Python
"""Integration and submission-contract tests for final_pipeline.py."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
import numpy as np
|
|
|
|
from benchmark_grouped import FAULT_LABELS, FREQ_COLS, NOT_APPLICABLE
|
|
from final_pipeline import (
|
|
OOD_OK_TO_UNKNOWN_THRESHOLD_MV,
|
|
OOD_OK_TO_UNKNOWN_RATIO_THRESHOLD,
|
|
apply_ood_override,
|
|
run_pipeline,
|
|
validate_inference_data,
|
|
validate_submission,
|
|
)
|
|
|
|
|
|
REPO_DIR = Path(__file__).resolve().parent
|
|
|
|
|
|
class FinalPipelineIntegrationTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
cls.val = pd.read_csv(REPO_DIR / "val.csv").reset_index(drop=True)
|
|
cls.test = pd.read_csv(REPO_DIR / "test.csv").reset_index(drop=True)
|
|
cls.sample = pd.read_csv(REPO_DIR / "sample_submit.csv").reset_index(drop=True)
|
|
cls.submission, cls.diagnostics = run_pipeline(
|
|
val=cls.val,
|
|
test=cls.test,
|
|
sample_submit=cls.sample,
|
|
random_state=42,
|
|
n_jobs=1,
|
|
)
|
|
|
|
def test_submission_contract_is_valid(self) -> None:
|
|
validate_submission(self.submission, self.test)
|
|
self.assertEqual(len(self.submission), 600)
|
|
|
|
def test_severity_business_rule_is_enforced(self) -> None:
|
|
fault = self.submission["label"].isin(FAULT_LABELS)
|
|
self.assertTrue(
|
|
self.submission.loc[~fault, "severity"].eq(NOT_APPLICABLE).all()
|
|
)
|
|
self.assertFalse(
|
|
self.submission.loc[fault, "severity"].eq(NOT_APPLICABLE).any()
|
|
)
|
|
|
|
def test_diagnostics_are_complete_for_the_application(self) -> None:
|
|
self.assertEqual(len(self.diagnostics), len(self.test))
|
|
classifier = self.diagnostics["decision_source"].eq("classifier")
|
|
ood = self.diagnostics["decision_source"].eq("ood_override")
|
|
self.assertTrue(
|
|
self.diagnostics.loc[classifier, "label_confidence"].between(0, 1).all()
|
|
)
|
|
self.assertTrue(self.diagnostics.loc[ood, "label_confidence"].isna().all())
|
|
self.assertTrue(self.diagnostics["label_margin"].between(0, 1).all())
|
|
self.assertTrue(
|
|
self.diagnostics["anomaly_score_mean_abs_mv"].ge(0).all()
|
|
)
|
|
self.assertEqual(
|
|
int(self.diagnostics["missing_spectral_cells"].sum()),
|
|
int(self.test.filter(regex=r"^mV_").isna().sum().sum()),
|
|
)
|
|
|
|
def test_validator_rejects_wrong_non_fault_severity(self) -> None:
|
|
broken = self.submission.copy()
|
|
position = broken.index[~broken["label"].isin(FAULT_LABELS)][0]
|
|
broken.loc[position, "severity"] = "male"
|
|
with self.assertRaisesRegex(ValueError, "nie_dotyczy"):
|
|
validate_submission(broken, self.test)
|
|
|
|
|
|
class InferenceInputTests(unittest.TestCase):
|
|
def test_incomplete_engine_is_rejected(self) -> None:
|
|
test = pd.read_csv(REPO_DIR / "test.csv").reset_index(drop=True)
|
|
incomplete = test.drop(index=test.index[0]).reset_index(drop=True)
|
|
with self.assertRaisesRegex(ValueError, "Incomplete"):
|
|
validate_inference_data(incomplete)
|
|
|
|
|
|
class OODRuleTests(unittest.TestCase):
|
|
def test_only_anomalous_ok_is_overridden(self) -> None:
|
|
labels = np.asarray(["ok", "ok", "ok", "pompa"], dtype=object)
|
|
features = np.zeros((4, 2 * len(FREQ_COLS)), dtype=float)
|
|
features[2, len(FREQ_COLS) : 2 * len(FREQ_COLS)] = (
|
|
OOD_OK_TO_UNKNOWN_THRESHOLD_MV + 1.0
|
|
)
|
|
features[3, len(FREQ_COLS) : 2 * len(FREQ_COLS)] = (
|
|
OOD_OK_TO_UNKNOWN_THRESHOLD_MV + 10.0
|
|
)
|
|
predicted, overridden, _, ratio = apply_ood_override(
|
|
labels,
|
|
features,
|
|
np.asarray(["engine_a", "engine_a", "engine_a", "engine_b"]),
|
|
)
|
|
self.assertGreater(ratio[2], OOD_OK_TO_UNKNOWN_RATIO_THRESHOLD)
|
|
self.assertEqual(predicted.tolist(), ["ok", "ok", "unknown", "pompa"])
|
|
self.assertEqual(overridden.tolist(), [False, False, True, False])
|
|
|
|
def test_globally_noisy_engine_is_not_treated_as_isolated_ood(self) -> None:
|
|
labels = np.asarray(["ok", "ok", "ok"], dtype=object)
|
|
features = np.zeros((3, 2 * len(FREQ_COLS)), dtype=float)
|
|
features[:, len(FREQ_COLS) : 2 * len(FREQ_COLS)] = 10.0
|
|
predicted, overridden, score, ratio = apply_ood_override(
|
|
labels,
|
|
features,
|
|
np.asarray(["noisy", "noisy", "noisy"]),
|
|
)
|
|
self.assertTrue((score > OOD_OK_TO_UNKNOWN_THRESHOLD_MV).all())
|
|
self.assertTrue((ratio == 1.0).all())
|
|
self.assertEqual(predicted.tolist(), labels.tolist())
|
|
self.assertFalse(overridden.any())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|