150 lines
12 KiB
PHP
150 lines
12 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use app\common\service\prescriptionai\PrescriptionAiStatistics;
|
|
|
|
require dirname(__DIR__) . '/app/common/service/prescriptionai/PrescriptionAiStatistics.php';
|
|
|
|
$checks = 0;
|
|
function statisticsExpect(bool $condition, string $message): void
|
|
{
|
|
global $checks;
|
|
$checks++;
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
}
|
|
|
|
function statisticsNear($actual, float $expected, string $message): void
|
|
{
|
|
statisticsExpect(is_numeric($actual) && abs((float) $actual - $expected) < 1.0e-10, $message);
|
|
}
|
|
|
|
$record = static fn ($event, string $model, $score, array $extra = []): array => array_replace([
|
|
'event_id' => $event, 'patient_id' => 'patient-' . $event, 'doctor_id' => 7,
|
|
'model_key' => $model, 'baseline_eligible' => true,
|
|
'model_version' => $model . '-fixture-v1', 'prompt_version' => 'fixture-p1', 'dictionary_version' => 'fixture-d1',
|
|
'comparison' => ['status' => 'comparable', 'score' => $score, 'algorithm_version' => 'fixture-a1'],
|
|
], $extra);
|
|
$review = static fn (string $outcome, array $extra = []): array => array_replace([
|
|
'status' => 'completed', 'independent' => true, 'outcome' => $outcome,
|
|
'sampling_method' => 'random', 'disputed' => false,
|
|
], $extra);
|
|
|
|
$empty = PrescriptionAiStatistics::summarize([]);
|
|
statisticsExpect($empty['total_events'] === 0 && $empty['patient_count'] === 0, 'No input means no invented events');
|
|
statisticsExpect($empty['models']['qwen']['mean'] === null && $empty['models']['qwen']['coverage_percent'] === null, 'Empty score and denominator are unknown, never zero percent');
|
|
statisticsExpect($empty['reviews']['qualification_rate'] === null && $empty['reviews']['status'] === 'no_samples', 'No expert reviews means no fabricated qualification rate');
|
|
|
|
$rows = [
|
|
$record(1, 'qwen', 0), $record(1, 'openai', 60),
|
|
$record(2, 'qwen', 80, ['patient_id' => 'patient-1']),
|
|
$record(2, 'openai', null, ['patient_id' => 'patient-1', 'comparison' => ['status' => 'not_comparable', 'score' => null, 'reason_code' => 'model_failed']]),
|
|
$record(3, 'openai', 90, ['baseline_eligible' => false, 'exclusion_reason' => 'future_information']),
|
|
['event_id' => 4, 'patient_id' => 'patient-4'],
|
|
];
|
|
$summary = PrescriptionAiStatistics::summarize($rows);
|
|
statisticsExpect($summary['total_events'] === 4 && $summary['patient_count'] === 3 && $summary['repeated_patient_events'] === 1, 'Count events and unique patients rather than result rows');
|
|
statisticsExpect($summary['models']['qwen']['valid_count'] === 2 && $summary['models']['openai']['valid_count'] === 1, 'Each model has its own valid denominator');
|
|
statisticsNear($summary['models']['qwen']['coverage_percent'], 50.0, 'Qwen coverage uses all eligible events, including failures');
|
|
statisticsNear($summary['models']['openai']['coverage_percent'], 25.0, 'OpenAI coverage includes missing results in N');
|
|
statisticsNear($summary['models']['qwen']['mean'], 40.0, 'Genuine zero is a valid score included in the mean');
|
|
statisticsNear($summary['models']['qwen']['median'], 40.0, 'Even median uses the two middle original values');
|
|
statisticsExpect($summary['models']['qwen']['exclusion_reasons'] === ['missing_result' => 2], 'Missing model output is explicitly counted');
|
|
statisticsExpect($summary['models']['openai']['exclusion_reasons'] === ['future_information' => 1, 'missing_result' => 1, 'model_failed' => 1], 'Failure and fairness exclusions remain distinct');
|
|
statisticsExpect($summary['paired_count'] === 1 && $summary['paired_strata'][0]['count'] === 1, 'Paired comparison uses only events with both valid models');
|
|
statisticsNear($summary['paired_strata'][0]['qwen']['mean'], 0.0, 'Paired qwen mean does not use unpaired events');
|
|
statisticsNear($summary['paired_strata'][0]['openai']['mean'], 60.0, 'Paired openai mean uses the same event');
|
|
statisticsExpect($summary['reviews']['qualification_rate'] === null, 'AI agreement never becomes expert review qualification');
|
|
statisticsExpect($summary['models']['qwen']['sample_status'] === 'insufficient_sample', 'Small sample status is explicit, with no physician quality ranking');
|
|
|
|
$deduped = PrescriptionAiStatistics::summarize(array_merge($rows, [$rows[0], $rows[1], $rows[2]]));
|
|
statisticsExpect($deduped === $summary, 'Request retries and duplicate joins do not add samples');
|
|
$reverse = PrescriptionAiStatistics::summarize(array_reverse($rows));
|
|
statisticsExpect($reverse === $summary, 'Result arrival order does not alter the summary');
|
|
$conflict = PrescriptionAiStatistics::summarize([$record(1, 'qwen', 10), $record(1, 'qwen', 99), $record(1, 'openai', 80)]);
|
|
statisticsExpect($conflict['total_events'] === 1 && $conflict['models']['qwen']['valid_count'] === 0, 'Conflicting regenerated baselines cannot choose the favorable result');
|
|
statisticsExpect($conflict['models']['qwen']['exclusion_reasons'] === ['duplicate_baseline_conflict' => 1], 'Ambiguous frozen baseline is reported');
|
|
statisticsExpect($conflict['paired_count'] === 0, 'Conflicting baseline never enters paired comparison');
|
|
|
|
$fairness = PrescriptionAiStatistics::summarize([
|
|
$record(1, 'qwen', 100, ['baseline_eligible' => false, 'exclusion_reason' => 'non_independent']),
|
|
$record(2, 'qwen', 100, ['baseline_eligible' => false, 'exclusion_reason' => 'ai_assisted_revision']),
|
|
$record(3, 'qwen', 100, ['baseline_eligible' => false, 'exclusion_reason' => 'insufficient_data']),
|
|
$record(4, 'qwen', 100, ['baseline_eligible' => 1]),
|
|
$record(5, 'qwen', 100, ['baseline_eligible' => true, 'exclusion_reason' => 'future_information']),
|
|
]);
|
|
statisticsExpect($fairness['models']['qwen']['valid_count'] === 0 && $fairness['models']['qwen']['excluded_count'] === 5, 'Only explicit baseline qualification and no exclusion permit score aggregation');
|
|
statisticsExpect(count($fairness['models']['qwen']['exclusion_reasons']) === 5, 'Different baseline exclusions remain separately visible');
|
|
|
|
foreach ([null, '', true, false, [], -1, 101, INF, -INF, NAN, '1e9999'] as $score) {
|
|
$invalid = PrescriptionAiStatistics::summarize([$record(1, 'qwen', $score)]);
|
|
statisticsExpect($invalid['models']['qwen']['mean'] === null, 'Invalid score cannot become a number');
|
|
statisticsExpect($invalid['models']['qwen']['exclusion_reasons'] === ['invalid_score' => 1], 'Invalid score reason is explicit');
|
|
json_encode($invalid, JSON_THROW_ON_ERROR);
|
|
}
|
|
$invalidAlgorithm = PrescriptionAiStatistics::summarize([$record(1, 'qwen', 100, ['comparison' => ['status' => 'comparable', 'score' => 100]])]);
|
|
statisticsExpect($invalidAlgorithm['models']['qwen']['exclusion_reasons'] === ['missing_algorithm_version' => 1], 'Unversioned scores cannot enter baseline summaries');
|
|
$precision = PrescriptionAiStatistics::summarize([$record(1, 'qwen', 12.3456), $record(2, 'qwen', '78.9012'), $record(3, 'qwen', 90.0)]);
|
|
statisticsNear($precision['models']['qwen']['mean'], (12.3456 + 78.9012 + 90.0) / 3, 'Means preserve unrounded stored scores');
|
|
statisticsNear($precision['models']['qwen']['median'], 78.9012, 'Odd median is the exact middle score');
|
|
|
|
$mixedVersions = PrescriptionAiStatistics::summarize([
|
|
$record(1, 'qwen', 10), $record(1, 'openai', 15),
|
|
$record(2, 'qwen', 90, ['model_version' => 'qwen-fixture-v2']), $record(2, 'openai', 85),
|
|
]);
|
|
statisticsExpect($mixedVersions['models']['qwen']['mean'] === null && count($mixedVersions['models']['qwen']['strata']) === 2, 'Model version changes remain separate, with no silent combined mean');
|
|
statisticsExpect($mixedVersions['models']['qwen']['aggregation_status'] === 'stratified_versions', 'Client is told to display per-version summaries');
|
|
statisticsExpect($mixedVersions['paired_count'] === 2 && count($mixedVersions['paired_strata']) === 2, 'Paired sample counts also retain their version strata');
|
|
$algorithmChange = PrescriptionAiStatistics::summarize([
|
|
$record(1, 'qwen', 10), $record(2, 'qwen', 20, ['comparison' => ['status' => 'comparable', 'score' => 20, 'algorithm_version' => 'fixture-a2']]),
|
|
]);
|
|
statisticsExpect(count($algorithmChange['models']['qwen']['strata']) === 2, 'Algorithm upgrades create their own strata');
|
|
$binRows = [];
|
|
foreach ([0, 19.999, 20, 39.999, 40, 59.999, 60, 79.999, 80, 100] as $index => $score) {
|
|
$binRows[] = $record($index + 1, 'qwen', $score);
|
|
}
|
|
$bins = PrescriptionAiStatistics::summarize($binRows);
|
|
statisticsExpect(array_values($bins['models']['qwen']['distribution']) === [2, 2, 2, 2, 2], 'Distribution bin boundaries count zero and 100 correctly');
|
|
|
|
$identityConflict = PrescriptionAiStatistics::summarize([
|
|
$record(1, 'qwen', 10), $record(1, 'openai', 90, ['patient_id' => 'someone-else']),
|
|
]);
|
|
statisticsExpect($identityConflict['unknown_patient_events'] === 1 && $identityConflict['models']['qwen']['valid_count'] === 0, 'Conflicting event-patient binding cannot count as a valid baseline');
|
|
$invalidRows = PrescriptionAiStatistics::summarize([null, [], ['event_id' => 0], ['event_id' => false], $record(1, 'qwen', 10)]);
|
|
statisticsExpect($invalidRows['total_events'] === 1 && $invalidRows['invalid_row_count'] === 4, 'Malformed event rows are reported rather than counted as unique cases');
|
|
|
|
$reviewRows = [
|
|
$record(1, 'qwen', 10, ['review' => $review('qualified')]),
|
|
$record(1, 'openai', 90, ['review' => $review('qualified')]),
|
|
$record(2, 'qwen', 20, ['review' => $review('needs_revision')]),
|
|
$record(3, 'qwen', 30, ['review' => $review('unqualified')]),
|
|
$record(4, 'qwen', 40, ['review' => $review('not_evaluable')]),
|
|
$record(5, 'qwen', 50, ['review' => $review('qualified', ['status' => 'pending'])]),
|
|
$record(6, 'qwen', 60),
|
|
];
|
|
$reviews = PrescriptionAiStatistics::summarize($reviewRows)['reviews'];
|
|
statisticsExpect($reviews['reviewed_events'] === 5 && $reviews['unreviewed_events'] === 1, 'Review records deduplicate by event across model rows');
|
|
statisticsExpect($reviews['evaluable_count'] === 3 && $reviews['qualified_count'] === 1, 'Review denominator includes needs_revision and unqualified');
|
|
statisticsNear($reviews['qualification_rate'], 100.0 / 3.0, 'Expert rate only uses actual completed independent evaluable reviews');
|
|
statisticsNear($reviews['sampling_coverage_percent'], 500.0 / 6.0, 'Review sampling coverage uses all in-scope events');
|
|
statisticsExpect($reviews['exclusion_reasons'] === ['review_not_completed' => 1, 'review_not_evaluable' => 1], 'Unevaluable and incomplete review counts stay visible');
|
|
statisticsExpect($reviews['confidence_interval'] === null, 'No unsupported independence-based confidence interval is invented');
|
|
$separateReviews = PrescriptionAiStatistics::summarize([
|
|
$record(1, 'qwen', 20, ['review' => $review('qualified')]),
|
|
$record(2, 'qwen', 90, ['review' => $review('unqualified', ['sampling_method' => 'risk_directed'])]),
|
|
]);
|
|
statisticsExpect($separateReviews['reviews']['qualification_rate'] === null && count($separateReviews['reviews']['sampling_groups']) === 2, 'Targeted and representative reviews are never mixed into an overall qualification rate');
|
|
$excludedReviews = PrescriptionAiStatistics::summarize([
|
|
$record(1, 'qwen', 50, ['review' => $review('qualified', ['independent' => false])]),
|
|
$record(2, 'qwen', 50, ['review' => $review('qualified', ['disputed' => true])]),
|
|
$record(3, 'qwen', 50, ['review' => $review('qualified', ['sampling_method' => ''])]),
|
|
$record(4, 'qwen', 50, ['review' => $review('qualified')]),
|
|
$record(4, 'openai', 50, ['review' => $review('unqualified')]),
|
|
]);
|
|
statisticsExpect($excludedReviews['reviews']['qualification_rate'] === null && $excludedReviews['reviews']['evaluable_count'] === 0, 'Non-independent, disputed, unclassified and conflicting reviews cannot create a qualification rate');
|
|
statisticsExpect($excludedReviews['models']['qwen']['valid_count'] === 4, 'Review disagreements do not alter structural AI comparison scores');
|
|
|
|
echo 'PRESCRIPTION_AI_STATISTICS_TEST_OK ' . $checks . " checks\n";
|