chat/backend/tests/llm_100_round_regression.php

<?php

declare(strict_types=1);

require dirname(__DIR__) . '/vendor/autoload.php';

use app\service\DifyService;
use app\service\OpenAIService;

$app = new think\App(dirname(__DIR__) . DIRECTORY_SEPARATOR);
$app->initialize();

$model = OpenAIService::getLanguageModel();
$cases = [];
$chineseExactReplyPrompt = "\u{8BF7}\u{53EA}\u{56DE}\u{590D}\u{8FD9}\u{4E2A}\u{5B57}\u{7B26}\u{4E32}\u{FF0C}\u{4E0D}\u{8981}\u{6DFB}\u{52A0}\u{4EFB}\u{4F55}\u{5176}\u{4ED6}\u{5185}\u{5BB9}\u{FF1A}";

$buildBlockingCase = static function (string $name, string $prompt, callable $assertion) use (&$cases, $model): void {
    $cases[] = [
        'name' => $name,
        'run' => static function () use ($model, $prompt, $assertion): array {
            if (($model->provider ?? '') === 'dify') {
                $response = DifyService::chat(
                    $model,
                    $prompt,
                    [],
                    null,
                    'llm-regression-blocking-' . bin2hex(random_bytes(4))
                );
                $answer = trim((string) ($response['answer'] ?? ''));
            } else {
                $response = OpenAIService::chat($model, [
                    ['role' => 'user', 'content' => $prompt],
                ]);
                $answer = OpenAIService::extractMessageContent($response);
            }

            return $assertion($answer);
        },
    ];
};

$buildStreamingCase = static function (string $name, string $prompt, callable $assertion) use (&$cases, $model): void {
    $cases[] = [
        'name' => $name,
        'run' => static function () use ($model, $prompt, $assertion): array {
            $answer = '';
            $streamError = null;

            if (($model->provider ?? '') === 'dify') {
                DifyService::streamChat(
                    $model,
                    $prompt,
                    [],
                    null,
                    'llm-regression-stream-' . bin2hex(random_bytes(4)),
                    static function (string $chunk) use (&$answer): void {
                        $answer .= $chunk;
                    },
                    static function (): void {
                    },
                    static function (string $message) use (&$streamError): void {
                        $streamError = $message;
                    }
                );
            } else {
                OpenAIService::streamChat(
                    $model,
                    [['role' => 'user', 'content' => $prompt]],
                    static function (array $chunk) use (&$answer): void {
                        $answer .= OpenAIService::extractStreamDelta($chunk);
                    },
                    static function (string $message) use (&$streamError): void {
                        $streamError = $message;
                    }
                );
            }

            if ($streamError !== null) {
                return [false, $streamError];
            }

            return $assertion(trim($answer));
        },
    ];
};

$contains = static function (string $needle): callable {
    return static function (string $answer) use ($needle): array {
        $ok = $answer !== '' && str_contains($answer, $needle);
        return [$ok, 'expected token=' . $needle . ' actual=' . $answer];
    };
};

for ($index = 1; $index <= 25; $index++) {
    $token = 'BLOCK-CN-' . str_pad((string) $index, 2, '0', STR_PAD_LEFT);
    $buildBlockingCase(
        'blocking_cn_' . $index,
        $chineseExactReplyPrompt . $token,
        $contains($token)
    );
}

for ($index = 1; $index <= 25; $index++) {
    $token = 'BLOCK-EN-' . str_pad((string) $index, 2, '0', STR_PAD_LEFT);
    $buildBlockingCase(
        'blocking_en_' . $index,
        'Reply with this exact token only and nothing else: ' . $token,
        $contains($token)
    );
}

for ($index = 1; $index <= 25; $index++) {
    $token = 'STREAM-CN-' . str_pad((string) $index, 2, '0', STR_PAD_LEFT);
    $buildStreamingCase(
        'streaming_cn_' . $index,
        $chineseExactReplyPrompt . $token,
        $contains($token)
    );
}

for ($index = 1; $index <= 25; $index++) {
    $token = 'STREAM-EN-' . str_pad((string) $index, 2, '0', STR_PAD_LEFT);
    $buildStreamingCase(
        'streaming_en_' . $index,
        'Reply with this exact token only and nothing else: ' . $token,
        $contains($token)
    );
}

if (count($cases) !== 100) {
    fwrite(STDERR, 'Test definition error: expected 100 cases, got ' . count($cases) . PHP_EOL);
    exit(2);
}

$passed = 0;
$failures = [];

foreach ($cases as $index => $case) {
    try {
        [$ok, $detail] = $case['run']();
    } catch (Throwable $exception) {
        $ok = false;
        $detail = get_class($exception) . ': ' . $exception->getMessage();
    }

    $number = $index + 1;
    if ($ok) {
        $passed++;
        printf("ROUND %03d PASS %s\n", $number, $case['name']);
        continue;
    }

    $failures[] = [$number, $case['name'], $detail];
    printf("ROUND %03d FAIL %s\n  %s\n", $number, $case['name'], $detail);
}

printf("RESULT %d/100 passed; %d failed\n", $passed, count($failures));
exit($failures === [] ? 0 : 1);