更新
This commit is contained in:
Your Name
@@ -0,0 +1,335 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use app\controller\api\Chat;
|
||||
use app\service\AgentCatalog;
|
||||
|
||||
$chatReflection = new ReflectionClass(Chat::class);
|
||||
$chat = $chatReflection->newInstanceWithoutConstructor();
|
||||
$resolveTurn = $chatReflection->getMethod('resolveAgentImageTurn');
|
||||
$resolveTurn->setAccessible(true);
|
||||
$detectFabricatedImage = $chatReflection->getMethod('containsFabricatedAgentImage');
|
||||
$detectFabricatedImage->setAccessible(true);
|
||||
|
||||
$user = static function (string $content, array $attachments = []): array {
|
||||
return ['role' => 'user', 'content' => $content, 'attachments' => $attachments];
|
||||
};
|
||||
$assistant = static function (string $content, array $attachments = []): array {
|
||||
return ['role' => 'assistant', 'content' => $content, 'attachments' => $attachments];
|
||||
};
|
||||
|
||||
$generatedImage = [[
|
||||
'type' => 'image',
|
||||
'url' => '/api/uploads/agent-regression.png',
|
||||
'generation_prompt' => 'A photorealistic brown horse standing in a green grassland.',
|
||||
]];
|
||||
|
||||
$cases = [];
|
||||
$addRouteCase = static function (
|
||||
string $name,
|
||||
string $content,
|
||||
array $priorHistory,
|
||||
string $mode,
|
||||
?string $subject = null
|
||||
) use (&$cases, $chat, $resolveTurn, $user): void {
|
||||
$cases[] = [
|
||||
'name' => $name,
|
||||
'run' => static function () use (
|
||||
$chat,
|
||||
$resolveTurn,
|
||||
$user,
|
||||
$content,
|
||||
$priorHistory,
|
||||
$mode,
|
||||
$subject
|
||||
): array {
|
||||
$history = array_merge($priorHistory, [$user($content)]);
|
||||
$actual = $resolveTurn->invoke($chat, $content, $history);
|
||||
$expectedModes = [
|
||||
'text' => ['allowed' => false, 'revision' => false, 'independent' => false],
|
||||
'new' => ['allowed' => true, 'revision' => false, 'independent' => true],
|
||||
'revision' => ['allowed' => true, 'revision' => true, 'independent' => false],
|
||||
];
|
||||
$expected = $expectedModes[$mode];
|
||||
$checks = [
|
||||
($actual['allowed'] ?? null) === $expected['allowed'],
|
||||
($actual['revision'] ?? null) === $expected['revision'],
|
||||
($actual['independent'] ?? null) === $expected['independent'],
|
||||
];
|
||||
if ($subject !== null) {
|
||||
$checks[] = ($actual['subject'] ?? '') === $subject;
|
||||
}
|
||||
|
||||
return [
|
||||
!in_array(false, $checks, true),
|
||||
'expected=' . json_encode(array_merge($expected, $subject === null ? [] : ['subject' => $subject]), JSON_UNESCAPED_UNICODE)
|
||||
. ' actual=' . json_encode($actual, JSON_UNESCAPED_UNICODE),
|
||||
];
|
||||
},
|
||||
];
|
||||
};
|
||||
$addStaticCase = static function (string $name, callable $run) use (&$cases): void {
|
||||
$cases[] = ['name' => $name, 'run' => $run];
|
||||
};
|
||||
|
||||
// 01-10: direct image creation expressed in normal spoken Chinese.
|
||||
foreach ([
|
||||
['生成一张雨夜城市图片', '生成一张雨夜城市图片'],
|
||||
['帮我画一只橘猫', '帮我画一只橘猫'],
|
||||
['来张极光壁纸', '来张极光壁纸'],
|
||||
['给我做个电商产品主图', '给我做个电商产品主图'],
|
||||
['设计一个极简logo', '设计一个极简logo'],
|
||||
['画一下我家门口的梧桐树', '画一下我家门口的梧桐树'],
|
||||
['给整一张海边日落', '给整一张海边日落'],
|
||||
['帮我出个头像', '帮我出个头像'],
|
||||
['来个赛博城市夜景', '来个赛博城市夜景'],
|
||||
['整个古风书房看看', '整个古风书房看看'],
|
||||
] as [$name, $content]) {
|
||||
$addRouteCase($name, $content, [], 'new');
|
||||
}
|
||||
|
||||
// 11-20: image-related text questions and explicit deferrals must stay on the LLM.
|
||||
foreach ([
|
||||
'你会画图吗',
|
||||
'能不能生成图片',
|
||||
'不要生成图，只描述一座雪山',
|
||||
'先给三个海报方向，别出图',
|
||||
'怎么写生图提示词',
|
||||
'解释一下图片生成原理',
|
||||
'分析这张图片',
|
||||
'图片里是谁',
|
||||
'给我一个绘图教程',
|
||||
'先讨论构图，确认后再画',
|
||||
] as $content) {
|
||||
$addRouteCase($content, $content, [], 'text');
|
||||
}
|
||||
|
||||
// 21-30: contextual requests should infer the single subject instead of lecturing about it.
|
||||
$contextSubjects = [
|
||||
['熊猫', '我想看看'],
|
||||
['长颈鹿', '给我看看'],
|
||||
['金字塔', '长啥样啊'],
|
||||
['雪豹', '也让我瞧瞧'],
|
||||
['极光', '有照片吗'],
|
||||
['火山', '给瞅一眼'],
|
||||
['北极熊', '我想看一下它'],
|
||||
['故宫', '展示一下吧'],
|
||||
['空间站', '能看到吗'],
|
||||
['珊瑚', '看看呗'],
|
||||
];
|
||||
foreach ($contextSubjects as [$subject, $content]) {
|
||||
$history = [
|
||||
$user('你知道' . $subject . '吗'),
|
||||
$assistant($subject . '是一种很有代表性的对象，外观很有特点。'),
|
||||
];
|
||||
$addRouteCase($subject . ' / ' . $content, $content, $history, 'new', $subject);
|
||||
}
|
||||
|
||||
// 31-40: a newly introduced topic must replace the older subject.
|
||||
$topicSwitches = [
|
||||
['兵马俑', '长城', '也给我看看'],
|
||||
['月球', '火星', '那也看看'],
|
||||
['长城', '兵马俑', '这个也想看看'],
|
||||
['蓝鲸', '鲸鲨', '那它长什么样'],
|
||||
['长城', '金字塔', '我也想瞧瞧'],
|
||||
['猎豹', '雪豹', '也来张它的'],
|
||||
['航空母舰', '空间站', '它有图吗'],
|
||||
['颐和园', '故宫', '看看刚说的那个'],
|
||||
['银河系', '黑洞', '那玩意儿长啥样'],
|
||||
['海草', '珊瑚', '也展示一下呗'],
|
||||
];
|
||||
foreach ($topicSwitches as [$oldSubject, $newSubject, $content]) {
|
||||
$history = [
|
||||
$user('你知道' . $oldSubject . '吗'),
|
||||
$assistant($oldSubject . '很有名。'),
|
||||
$user('那' . $newSubject . '呢？'),
|
||||
$assistant($newSubject . '也很有代表性，外观特点鲜明。'),
|
||||
];
|
||||
$addRouteCase($oldSubject . ' -> ' . $newSubject . ' / ' . $content, $content, $history, 'new', $newSubject);
|
||||
}
|
||||
|
||||
// 41-50: terse visual feedback after a generated image means image revision.
|
||||
foreach ([
|
||||
'鬃毛再长一点',
|
||||
'亮堂点',
|
||||
'别这么挤',
|
||||
'人物往左一点',
|
||||
'背景换成海边',
|
||||
'镜头拉远些',
|
||||
'颜色淡一点',
|
||||
'别这么塑料',
|
||||
'更有电影感',
|
||||
'字去掉',
|
||||
] as $content) {
|
||||
$addRouteCase($content, $content, [$assistant('已根据描述生成图片：', $generatedImage)], 'revision');
|
||||
}
|
||||
|
||||
// 51-60: questions about an existing image must not accidentally regenerate it.
|
||||
foreach ([
|
||||
'这是什么品种',
|
||||
'它为什么有角',
|
||||
'能分析下构图吗',
|
||||
'这图是真的吗',
|
||||
'你觉得好看吗',
|
||||
'保存在哪',
|
||||
'能下载吗',
|
||||
'看不懂',
|
||||
'这是什么地方',
|
||||
'为什么颜色这么暗',
|
||||
] as $content) {
|
||||
$addRouteCase($content, $content, [$assistant('已根据描述生成图片：', $generatedImage)], 'text');
|
||||
}
|
||||
|
||||
// 61-70: short confirmations are image actions only after a concrete visual plan.
|
||||
$plan = '海报可以做三个方向：方案一暖色纪实，方案二蓝色电影感，方案三黑白极简。我推荐方案二。你选哪个，我确认后就生成图片。';
|
||||
foreach ([
|
||||
'第二个挺好，就照它来',
|
||||
'那就第二个吧',
|
||||
'选3',
|
||||
'第三套',
|
||||
'就最后那个',
|
||||
'按第一个来',
|
||||
'听你的',
|
||||
'你推荐的那个',
|
||||
'开始干吧',
|
||||
'可以，出图',
|
||||
] as $content) {
|
||||
$addRouteCase($content, $content, [$assistant($plan)], 'new');
|
||||
}
|
||||
|
||||
// 71-80: confirmations after image analysis continue the existing generated image.
|
||||
$revisionAdvice = '这张图显得冷清，建议增加暖色灯光、拉近人物，并调整背景层次。需要我按这个调整方向重新生成一张吗？';
|
||||
foreach ([
|
||||
'那就这样改',
|
||||
'行，照办',
|
||||
'按这个调',
|
||||
'可以，动手',
|
||||
'照你说的优化',
|
||||
'嗯，就这么弄',
|
||||
'那就按这个方向调整',
|
||||
'好，修一下',
|
||||
'执行吧',
|
||||
'就这么办',
|
||||
] as $content) {
|
||||
$history = [
|
||||
$assistant('已根据描述生成图片：', $generatedImage),
|
||||
$user('为什么这张图看起来这么冷清？'),
|
||||
$assistant($revisionAdvice),
|
||||
];
|
||||
$addRouteCase($content, $content, $history, 'revision');
|
||||
}
|
||||
|
||||
// 81-90: pronouns, multiple entities, topic correction, and unrelated visibility wording.
|
||||
$addRouteCase('两个人 + 单数他', '我想看看他', [
|
||||
$user('你知道周杰伦和林俊杰吗'),
|
||||
$assistant('周杰伦和林俊杰都是歌手。'),
|
||||
], 'text');
|
||||
$addRouteCase('两个人 + 复数他们', '我想看看他们', [
|
||||
$user('你知道周杰伦和林俊杰吗'),
|
||||
$assistant('周杰伦和林俊杰都是歌手。'),
|
||||
], 'new', '周杰伦和林俊杰');
|
||||
$addRouteCase('无前文的他', '我想看看他', [], 'text');
|
||||
$addRouteCase('代码看不到', '我看不到', [
|
||||
$user('这段 PHP 代码为什么报错'),
|
||||
$assistant('请检查变量是否定义，并查看错误日志。'),
|
||||
], 'text');
|
||||
$addRouteCase('旧话题切换到长城', '也给我看看', [
|
||||
$user('你知道兵马俑吗'),
|
||||
$assistant('兵马俑是秦代陶俑群。'),
|
||||
$user('那长城呢'),
|
||||
$assistant('长城是中国古代防御工程。'),
|
||||
], 'new', '长城');
|
||||
$addRouteCase('两种动物 + 单数它', '我想看看它', [
|
||||
$user('你知道雪豹和猎豹吗'),
|
||||
$assistant('雪豹和猎豹是两种不同的猫科动物。'),
|
||||
], 'text');
|
||||
$addRouteCase('当前明确两个人', '我想看刘亦菲和胡歌', [], 'new', '刘亦菲和胡歌');
|
||||
$addRouteCase('两个对象 + 这个', '这个长什么样', [
|
||||
$user('你知道火星和月球吗'),
|
||||
$assistant('火星和月球都是常见天体。'),
|
||||
], 'text');
|
||||
$addRouteCase('两个人都看', '两个都给我看看', [
|
||||
$user('你知道刘亦菲和胡歌吗'),
|
||||
$assistant('刘亦菲和胡歌都是演员。'),
|
||||
], 'new', '刘亦菲和胡歌');
|
||||
$addRouteCase('纠正后的最新主体', '看看它', [
|
||||
$user('你知道牛吗'),
|
||||
$assistant('牛是常见家畜。'),
|
||||
$user('不是牛，我说的是马'),
|
||||
$assistant('明白，你说的是马。'),
|
||||
], 'new', '马');
|
||||
|
||||
// 91-100: private action envelope and fabricated-image safety boundaries.
|
||||
$addStaticCase('严格 JSON 动作', static function (): array {
|
||||
$actual = AgentCatalog::parseImageAction('{"action":"generate_image","prompt":"a red fox in snow"}');
|
||||
return [$actual !== null && $actual['prompt'] === 'a red fox in snow', json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
||||
});
|
||||
$addStaticCase('Markdown JSON 动作', static function (): array {
|
||||
$actual = AgentCatalog::parseImageAction("```json\n{\"action\":\"generate_image\",\"prompt\":\"a lighthouse at dusk\"}\n```");
|
||||
return [$actual !== null && $actual['prompt'] === 'a lighthouse at dusk', json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
||||
});
|
||||
$addStaticCase('短确认包裹动作', static function (): array {
|
||||
$actual = AgentCatalog::parseImageAction('好的，马上生成。 {"action":"generate_image","prompt":"a golden dragon"}');
|
||||
return [$actual !== null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
||||
});
|
||||
$addStaticCase('长执行说明包裹动作', static function (): array {
|
||||
$prefix = '我将为你执行图片生成，并根据刚才确认的主体、构图、环境、光线、镜头语言和材质细节创建一幅完整画面，现在开始生成。';
|
||||
$actual = AgentCatalog::parseImageAction($prefix . ' {"action":"generate_image","prompt":"a cinematic mountain village"}');
|
||||
return [$actual !== null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
||||
});
|
||||
$addStaticCase('示例动作不能执行', static function (): array {
|
||||
$actual = AgentCatalog::parseImageAction('例如可以输出：{"action":"generate_image","prompt":"example only"}');
|
||||
return [$actual === null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
||||
});
|
||||
$addStaticCase('错误动作不能执行', static function (): array {
|
||||
$actual = AgentCatalog::parseImageAction('{"action":"search_image","prompt":"a cat"}');
|
||||
return [$actual === null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
||||
});
|
||||
$addStaticCase('空提示词不能执行', static function (): array {
|
||||
$actual = AgentCatalog::parseImageAction('{"action":"generate_image","prompt":""}');
|
||||
return [$actual === null, json_encode($actual, JSON_UNESCAPED_UNICODE)];
|
||||
});
|
||||
$addStaticCase('伪造已生成外链图片', static function () use ($chat, $detectFabricatedImage): array {
|
||||
$actual = $detectFabricatedImage->invoke($chat, '已根据描述为你生成图片：');
|
||||
return [$actual === true, var_export($actual, true)];
|
||||
});
|
||||
$addStaticCase('普通引用外链图片', static function () use ($chat, $detectFabricatedImage): array {
|
||||
$actual = $detectFabricatedImage->invoke($chat, '参考资料中的示意图：');
|
||||
return [$actual === false, var_export($actual, true)];
|
||||
});
|
||||
$addStaticCase('明确延后生图', static function (): array {
|
||||
$actual = AgentCatalog::requestsImageGeneration('我想做海报，先谈方案，等我确认后再生成');
|
||||
return [$actual === false, var_export($actual, true)];
|
||||
});
|
||||
|
||||
if (count($cases) !== 100) {
|
||||
fwrite(STDERR, 'Test definition error: expected 100 cases, got ' . count($cases) . PHP_EOL);
|
||||
exit(2);
|
||||
}
|
||||
|
||||
$passed = 0;
|
||||
$failures = [];
|
||||
foreach ($cases as $index => $case) {
|
||||
try {
|
||||
[$ok, $detail] = $case['run']();
|
||||
} catch (Throwable $exception) {
|
||||
$ok = false;
|
||||
$detail = get_class($exception) . ': ' . $exception->getMessage();
|
||||
}
|
||||
|
||||
$number = $index + 1;
|
||||
if ($ok) {
|
||||
$passed++;
|
||||
printf("ROUND %03d PASS %s\n", $number, $case['name']);
|
||||
continue;
|
||||
}
|
||||
|
||||
$failures[] = [$number, $case['name'], $detail];
|
||||
printf("ROUND %03d FAIL %s\n %s\n", $number, $case['name'], $detail);
|
||||
}
|
||||
|
||||
printf("RESULT %d/100 passed; %d failed\n", $passed, count($failures));
|
||||
exit($failures === [] ? 0 : 1);
|
||||
Reference in New Issue
Block a user