336 lines
14 KiB
PHP
336 lines
14 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use app\controller\api\Chat;
|
|
use app\service\AgentCatalog;
|
|
|
|
$chatReflection = new ReflectionClass(Chat::class);
|
|
$chat = $chatReflection->newInstanceWithoutConstructor();
|
|
$resolveTurn = $chatReflection->getMethod('resolveAgentImageTurn');
|
|
$resolveTurn->setAccessible(true);
|
|
$detectFabricatedImage = $chatReflection->getMethod('containsFabricatedAgentImage');
|
|
$detectFabricatedImage->setAccessible(true);
|
|
|
|
$user = static function (string $content, array $attachments = []): array {
|
|
return ['role' => 'user', 'content' => $content, 'attachments' => $attachments];
|
|
};
|
|
$assistant = static function (string $content, array $attachments = []): array {
|
|
return ['role' => 'assistant', 'content' => $content, 'attachments' => $attachments];
|
|
};
|
|
|
|
$generatedImage = [[
|
|
'type' => 'image',
|
|
'url' => '/api/uploads/agent-regression.png',
|
|
'generation_prompt' => 'A photorealistic brown horse standing in a green grassland.',
|
|
]];
|
|
|
|
$cases = [];
|
|
$addRouteCase = static function (
|
|
string $name,
|
|
string $content,
|
|
array $priorHistory,
|
|
string $mode,
|
|
?string $subject = null
|
|
) use (&$cases, $chat, $resolveTurn, $user): void {
|
|
$cases[] = [
|
|
'name' => $name,
|
|
'run' => static function () use (
|
|
$chat,
|
|
$resolveTurn,
|
|
$user,
|
|
$content,
|
|
$priorHistory,
|
|
$mode,
|
|
$subject
|
|
): array {
|
|
$history = array_merge($priorHistory, [$user($content)]);
|
|
$actual = $resolveTurn->invoke($chat, $content, $history);
|
|
$expectedModes = [
|
|
'text' => ['allowed' => false, 'revision' => false, 'independent' => false],
|
|
'new' => ['allowed' => true, 'revision' => false, 'independent' => true],
|
|
'revision' => ['allowed' => true, 'revision' => true, 'independent' => false],
|
|
];
|
|
$expected = $expectedModes[$mode];
|
|
$checks = [
|
|
($actual['allowed'] ?? null) === $expected['allowed'],
|
|
($actual['revision'] ?? null) === $expected['revision'],
|
|
($actual['independent'] ?? null) === $expected['independent'],
|
|
];
|
|
if ($subject !== null) {
|
|
$checks[] = ($actual['subject'] ?? '') === $subject;
|
|
}
|
|
|
|
return [
|
|
!in_array(false, $checks, true),
|
|
'expected=' . json_encode(array_merge($expected, $subject === null ? [] : ['subject' => $subject]), JSON_UNESCAPED_UNICODE)
|
|
. ' actual=' . json_encode($actual, JSON_UNESCAPED_UNICODE),
|
|
];
|
|
},
|
|
];
|
|
};
|
|
$addStaticCase = static function (string $name, callable $run) use (&$cases): void {
|
|
$cases[] = ['name' => $name, 'run' => $run];
|
|
};
|
|
|
|
// 01-10: direct image creation expressed in normal spoken Chinese.
|
|
foreach ([
|
|
['生成一张雨夜城市图片', '生成一张雨夜城市图片'],
|
|
['帮我画一只橘猫', '帮我画一只橘猫'],
|
|
['来张极光壁纸', '来张极光壁纸'],
|
|
['给我做个电商产品主图', '给我做个电商产品主图'],
|
|
['设计一个极简logo', '设计一个极简logo'],
|
|
['画一下我家门口的梧桐树', '画一下我家门口的梧桐树'],
|
|
['给整一张海边日落', '给整一张海边日落'],
|
|
['帮我出个头像', '帮我出个头像'],
|
|
['来个赛博城市夜景', '来个赛博城市夜景'],
|
|
['整个古风书房看看', '整个古风书房看看'],
|
|
] as [$name, $content]) {
|
|
$addRouteCase($name, $content, [], 'new');
|
|
}
|
|
|
|
// 11-20: image-related text questions and explicit deferrals must stay on the LLM.
|
|
foreach ([
|
|
'你会画图吗',
|
|
'能不能生成图片',
|
|
'不要生成图，只描述一座雪山',
|
|
'先给三个海报方向，别出图',
|
|
'怎么写生图提示词',
|
|
'解释一下图片生成原理',
|
|
'分析这张图片',
|
|
'图片里是谁',
|
|
'给我一个绘图教程',
|
|
'先讨论构图，确认后再画',
|
|
] as $content) {
|
|
$addRouteCase($content, $content, [], 'text');
|
|
}
|
|
|
|
// 21-30: contextual requests should infer the single subject instead of lecturing about it.
|
|
$contextSubjects = [
|
|
['熊猫', '我想看看'],
|
|
['长颈鹿', '给我看看'],
|
|
['金字塔', '长啥样啊'],
|
|
['雪豹', '也让我瞧瞧'],
|
|
['极光', '有照片吗'],
|
|
['火山', '给瞅一眼'],
|
|
['北极熊', '我想看一下它'],
|
|
['故宫', '展示一下吧'],
|
|
['空间站', '能看到吗'],
|
|
['珊瑚', '看看呗'],
|
|
];
|
|
foreach ($contextSubjects as [$subject, $content]) {
|
|
$history = [
|
|
$user('你知道' . $subject . '吗'),
|
|
$assistant($subject . '是一种很有代表性的对象，外观很有特点。'),
|
|
];
|
|
$addRouteCase($subject . ' / ' . $content, $content, $history, 'new', $subject);
|
|
}
|
|
|
|
// 31-40: a newly introduced topic must replace the older subject.
|
|
$topicSwitches = [
|
|
['兵马俑', '长城', '也给我看看'],
|
|
['月球', '火星', '那也看看'],
|
|
['长城', '兵马俑', '这个也想看看'],
|
|
['蓝鲸', '鲸鲨', '那它长什么样'],
|
|
['长城', '金字塔', '我也想瞧瞧'],
|
|
['猎豹', '雪豹', '也来张它的'],
|
|
['航空母舰', '空间站', '它有图吗'],
|
|
['颐和园', '故宫', '看看刚说的那个'],
|
|
['银河系', '黑洞', '那玩意儿长啥样'],
|
|
['海草', '珊瑚', '也展示一下呗'],
|
|
];
|
|
foreach ($topicSwitches as [$oldSubject, $newSubject, $content]) {
|
|
$history = [
|
|
$user('你知道' . $oldSubject . '吗'),
|
|
$assistant($oldSubject . '很有名。'),
|
|
$user('那' . $newSubject . '呢？'),
|
|
$assistant($newSubject . '也很有代表性，外观特点鲜明。'),
|
|
];
|
|
$addRouteCase($oldSubject . ' -> ' . $newSubject . ' / ' . $content, $content, $history, 'new', $newSubject);
|
|
}
|
|
|
|
// 41-50: terse visual feedback after a generated image means image revision.
|
|
foreach ([
|
|
'鬃毛再长一点',
|
|
'亮堂点',
|
|
'别这么挤',
|
|
'人物往左一点',
|
|
'背景换成海边',
|
|
'镜头拉远些',
|
|
'颜色淡一点',
|
|
'别这么塑料',
|
|
'更有电影感',
|
|
'字去掉',
|
|
] as $content) {
|
|
$addRouteCase($content, $content, [$assistant('已根据描述生成图片：', $generatedImage)], 'revision');
|
|
}
|
|
|
|
// 51-60: questions about an existing image must not accidentally regenerate it.
|
|
foreach ([
|
|
'这是什么品种',
|
|
'它为什么有角',
|
|
'能分析下构图吗',
|
|
'这图是真的吗',
|
|
'你觉得好看吗',
|
|
'保存在哪',
|
|
'能下载吗',
|
|
'看不懂',
|
|
'这是什么地方',
|
|
'为什么颜色这么暗',
|
|
] as $content) {
|
|
$addRouteCase($content, $content, [$assistant('已根据描述生成图片：', $generatedImage)], 'text');
|
|
}
|
|
|
|
// 61-70: short confirmations are image actions only after a concrete visual plan.
|
|
$plan = '海报可以做三个方向：方案一暖色纪实，方案二蓝色电影感，方案三黑白极简。我推荐方案二。你选哪个，我确认后就生成图片。';
|
|
foreach ([
|
|
'第二个挺好，就照它来',
|
|
'那就第二个吧',
|
|
'选3',
|
|
'第三套',
|
|
'就最后那个',
|
|
'按第一个来',
|
|
'听你的',
|
|
'你推荐的那个',
|
|
'开始干吧',
|
|
'可以，出图',
|
|
] as $content) {
|
|
$addRouteCase($content, $content, [$assistant($plan)], 'new');
|
|
}
|
|
|
|
// 71-80: confirmations after image analysis continue the existing generated image.
|
|
$revisionAdvice = '这张图显得冷清，建议增加暖色灯光、拉近人物，并调整背景层次。需要我按这个调整方向重新生成一张吗？';
|
|
foreach ([
|
|
'那就这样改',
|
|
'行，照办',
|
|
'按这个调',
|
|
'可以，动手',
|
|
'照你说的优化',
|
|
'嗯，就这么弄',
|
|
'那就按这个方向调整',
|
|
'好，修一下',
|
|
'执行吧',
|
|
'就这么办',
|
|
] as $content) {
|
|
$history = [
|
|
$assistant('已根据描述生成图片：', $generatedImage),
|
|
$user('为什么这张图看起来这么冷清？'),
|
|
$assistant($revisionAdvice),
|
|
];
|
|
$addRouteCase($content, $content, $history, 'revision');
|
|
}
|
|
|
|
// 81-90: pronouns, multiple entities, topic correction, and unrelated visibility wording.
|
|
$addRouteCase('两个人 + 单数他', '我想看看他', [
|
|
$user('你知道周杰伦和林俊杰吗'),
|
|
$assistant('周杰伦和林俊杰都是歌手。'),
|
|
], 'text');
|
|
$addRouteCase('两个人 + 复数他们', '我想看看他们', [
|
|
$user('你知道周杰伦和林俊杰吗'),
|
|
$assistant('周杰伦和林俊杰都是歌手。'),
|
|
], 'new', '周杰伦和林俊杰');
|
|
$addRouteCase('无前文的他', '我想看看他', [], 'text');
|
|
$addRouteCase('代码看不到', '我看不到', [
|
|
$user('这段 PHP 代码为什么报错'),
|
|
$assistant('请检查变量是否定义，并查看错误日志。'),
|
|
], 'text');
|
|
$addRouteCase('旧话题切换到长城', '也给我看看', [
|
|
$user('你知道兵马俑吗'),
|
|
$assistant('兵马俑是秦代陶俑群。'),
|
|
$user('那长城呢'),
|
|
$assistant('长城是中国古代防御工程。'),
|
|
], 'new', '长城');
|
|
$addRouteCase('两种动物 + 单数它', '我想看看它', [
|
|
$user('你知道雪豹和猎豹吗'),
|
|
$assistant('雪豹和猎豹是两种不同的猫科动物。'),
|
|
], 'text');
|
|
$addRouteCase('当前明确两个人', '我想看刘亦菲和胡歌', [], 'new', '刘亦菲和胡歌');
|
|
$addRouteCase('两个对象 + 这个', '这个长什么样', [
|
|
$user('你知道火星和月球吗'),
|
|
$assistant('火星和月球都是常见天体。'),
|
|
], 'text');
|
|
$addRouteCase('两个人都看', '两个都给我看看', [
|
|
$user('你知道刘亦菲和胡歌吗'),
|
|
$assistant('刘亦菲和胡歌都是演员。'),
|
|
], 'new', '刘亦菲和胡歌');
|
|
$addRouteCase('纠正后的最新主体', '看看它', [
|
|
$user('你知道牛吗'),
|
|
$assistant('牛是常见家畜。'),
|
|
$user('不是牛，我说的是马'),
|
|
$assistant('明白，你说的是马。'),
|
|
], 'new', '马');
|
|
|
|
// 91-100: private action envelope and fabricated-image safety boundaries.
|
|
$addStaticCase('严格 JSON 动作', static function (): array {
|
|
$actual = AgentCatalog::parseImageAction('{"action":"generate_image","prompt":"a red fox in snow"}');
|
|
return [$actual !== null && $actual['prompt'] === 'a red fox in snow', json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
|
});
|
|
$addStaticCase('Markdown JSON 动作', static function (): array {
|
|
$actual = AgentCatalog::parseImageAction("```json\n{\"action\":\"generate_image\",\"prompt\":\"a lighthouse at dusk\"}\n```");
|
|
return [$actual !== null && $actual['prompt'] === 'a lighthouse at dusk', json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
|
});
|
|
$addStaticCase('短确认包裹动作', static function (): array {
|
|
$actual = AgentCatalog::parseImageAction('好的，马上生成。 {"action":"generate_image","prompt":"a golden dragon"}');
|
|
return [$actual !== null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
|
});
|
|
$addStaticCase('长执行说明包裹动作', static function (): array {
|
|
$prefix = '我将为你执行图片生成，并根据刚才确认的主体、构图、环境、光线、镜头语言和材质细节创建一幅完整画面，现在开始生成。';
|
|
$actual = AgentCatalog::parseImageAction($prefix . ' {"action":"generate_image","prompt":"a cinematic mountain village"}');
|
|
return [$actual !== null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
|
});
|
|
$addStaticCase('示例动作不能执行', static function (): array {
|
|
$actual = AgentCatalog::parseImageAction('例如可以输出：{"action":"generate_image","prompt":"example only"}');
|
|
return [$actual === null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
|
});
|
|
$addStaticCase('错误动作不能执行', static function (): array {
|
|
$actual = AgentCatalog::parseImageAction('{"action":"search_image","prompt":"a cat"}');
|
|
return [$actual === null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
|
});
|
|
$addStaticCase('空提示词不能执行', static function (): array {
|
|
$actual = AgentCatalog::parseImageAction('{"action":"generate_image","prompt":""}');
|
|
return [$actual === null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
|
});
|
|
$addStaticCase('伪造已生成外链图片', static function () use ($chat, $detectFabricatedImage): array {
|
|
$actual = $detectFabricatedImage->invoke($chat, '已根据描述为你生成图片：');
|
|
return [$actual === true, var_export($actual, true)];
|
|
});
|
|
$addStaticCase('普通引用外链图片', static function () use ($chat, $detectFabricatedImage): array {
|
|
$actual = $detectFabricatedImage->invoke($chat, '参考资料中的示意图：');
|
|
return [$actual === false, var_export($actual, true)];
|
|
});
|
|
$addStaticCase('明确延后生图', static function (): array {
|
|
$actual = AgentCatalog::requestsImageGeneration('我想做海报，先谈方案，等我确认后再生成');
|
|
return [$actual === false, var_export($actual, true)];
|
|
});
|
|
|
|
if (count($cases) !== 100) {
|
|
fwrite(STDERR, 'Test definition error: expected 100 cases, got ' . count($cases) . PHP_EOL);
|
|
exit(2);
|
|
}
|
|
|
|
$passed = 0;
|
|
$failures = [];
|
|
foreach ($cases as $index => $case) {
|
|
try {
|
|
[$ok, $detail] = $case['run']();
|
|
} catch (Throwable $exception) {
|
|
$ok = false;
|
|
$detail = get_class($exception) . ': ' . $exception->getMessage();
|
|
}
|
|
|
|
$number = $index + 1;
|
|
if ($ok) {
|
|
$passed++;
|
|
printf("ROUND %03d PASS %s\n", $number, $case['name']);
|
|
continue;
|
|
}
|
|
|
|
$failures[] = [$number, $case['name'], $detail];
|
|
printf("ROUND %03d FAIL %s\n %s\n", $number, $case['name'], $detail);
|
|
}
|
|
|
|
printf("RESULT %d/100 passed; %d failed\n", $passed, count($failures));
|
|
exit($failures === [] ? 0 : 1);
|