klarkxy
12bea84d8b
- 新增 llm_guidance 策略层，根据源码、判题结果与通过率评估完成度分级 - llm-review.php 改为服务端缓冲上游输出，仅下发白名单分类的固定模板 - 新增 llm_review 与 llm_review_rate_limit 表，实现结果缓存与账户/提交两级限流 - ceinfo 模板渲染编译器标出的源码行号/列号，reinfo 仅向高完成度提交展示结构化 diff - download.php 按完成度授权测试点下载，封堵白卷提交套取隐藏用例 - judge_client 在 ACM 非比赛场景下记录已通过测试点数，前端可显示进度 - 附 llm_guidance_policy_test 单测覆盖空白卷、注释卷、隐藏 SPJ、OJ_FULL_DIFF 等关键路径
161 lines
11 KiB
PHP
161 lines
11 KiB
PHP
<?php
|
|
require_once __DIR__ . '/../web/include/llm_guidance.inc.php';
|
|
|
|
function guidance_assert($condition, $message) {
|
|
if (!$condition) {
|
|
fwrite(STDERR, "FAIL: " . $message . PHP_EOL);
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
$blank = "#include <bits/stdc++.h>\nusing namespace std;\nint main(){ return 0; }";
|
|
$substantial = <<<'CPP'
|
|
#include <bits/stdc++.h>
|
|
using namespace std;
|
|
int main() {
|
|
int n, answer = 0;
|
|
cin >> n;
|
|
for (int i = 0; i < n; ++i) {
|
|
int value;
|
|
cin >> value;
|
|
answer += value;
|
|
}
|
|
cout << answer << "\n";
|
|
return 0;
|
|
}
|
|
CPP;
|
|
|
|
guidance_assert(llm_guidance_assess($substantial, 4, 1)['mode'] === 'hidden', 'AC must hide guidance');
|
|
guidance_assert(llm_guidance_assess($substantial, 3, 0)['mode'] === 'hidden', 'pending must hide guidance');
|
|
guidance_assert(llm_guidance_assess($substantial, 13, 0)['mode'] === 'hidden', 'test run must hide guidance');
|
|
|
|
$blank_wa = llm_guidance_assess($blank, 6, 0);
|
|
guidance_assert($blank_wa['mode'] === 'flowchart', 'blank/template WA must receive flowchart guidance');
|
|
guidance_assert($blank_wa['show_output_diff'] === false, 'blank/template WA must not expose expected-output diff');
|
|
|
|
$comment_only = "// for if while cin cout answer = 42\n/* int main() { cout << answer; } */";
|
|
guidance_assert(llm_guidance_assess($comment_only, 6, 0)['mode'] === 'flowchart', 'comment-only submissions must stay low completion');
|
|
|
|
$hash_comment_only = "# for if while input print answer = 1\n# if answer: print(answer)\n# more comments\n# still no code";
|
|
guidance_assert(llm_guidance_assess($hash_comment_only, 6, 0)['mode'] === 'flowchart', '# comment-only submissions must stay low completion');
|
|
|
|
$high_wa = llm_guidance_assess($substantial, 6, 0.85);
|
|
guidance_assert($high_wa['mode'] === 'output_diff', 'WA with a high judged pass rate must open output diff');
|
|
guidance_assert($high_wa['show_output_diff'] === true, 'high-completion WA must expose output diff');
|
|
|
|
$non_oi_wa = llm_guidance_assess($substantial, 6, 0);
|
|
guidance_assert($non_oi_wa['mode'] === 'focused_hint', 'pass_rate=0 must not unlock hidden output from source shape alone');
|
|
|
|
$high_pe = llm_guidance_assess($substantial, 5, 0);
|
|
guidance_assert($high_pe['mode'] === 'output_diff', 'substantial PE must open output diff');
|
|
$compact_pe = llm_guidance_assess('print(1)', 5, 0);
|
|
guidance_assert($compact_pe['mode'] === 'output_diff', 'judge-proven compact PE must open output diff');
|
|
|
|
$empty_output_info = "========[1.out]========\nExpected | Yours\n42 | \n========================\n";
|
|
$empty_attempt = llm_guidance_output_attempt($empty_output_info);
|
|
guidance_assert($empty_attempt['available'] && $empty_attempt['yours_chars'] === 0, 'empty judge output must be detected');
|
|
$padded_blank = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $empty_attempt);
|
|
guidance_assert($padded_blank['mode'] === 'flowchart', 'long source with empty output must be downgraded to flowchart');
|
|
guidance_assert($padded_blank['show_output_diff'] === false, 'long source with empty output must not expose expected output');
|
|
|
|
$trusted_empty_case = llm_guidance_refine_with_output_attempt($high_wa, 6, $empty_attempt);
|
|
guidance_assert($trusted_empty_case['mode'] === 'output_diff', 'one empty failing case must not erase authoritative overall pass progress');
|
|
|
|
$near_output_info = "========[1.out]========\nExpected | Yours\n12345 | 12346\n========================\n";
|
|
$near_attempt = llm_guidance_output_attempt($near_output_info);
|
|
$near_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $near_attempt);
|
|
guidance_assert($near_non_oi['mode'] === 'focused_hint', 'a near-looking hidden output must not unlock diff');
|
|
|
|
$oracle_output_info = "========[1.out]========\nExpected | Yours\n1000000000 | 0000000000\n========================\n";
|
|
$oracle_attempt = llm_guidance_output_attempt($oracle_output_info);
|
|
$oracle_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $oracle_attempt);
|
|
guidance_assert($oracle_non_oi['mode'] === 'focused_hint', 'a repeated-output guess must not unlock hidden expected output');
|
|
|
|
$two_case_info = "1.out\n--\n|Expected|Yours\n|--|--\n|1|0\n\n2.out\n--\n|Expected|Yours\n|--|--\n|2|3\n";
|
|
$two_case_names = llm_guidance_diff_testcase_names($two_case_info);
|
|
guidance_assert($two_case_names === array('1', '2'), 'only judge-recorded simple diff testcase names may be authorized');
|
|
guidance_assert(llm_guidance_testcase_base('../2.out') === null, 'testcase paths and traversal must be rejected');
|
|
guidance_assert(llm_guidance_testcase_base('2.out') === '2', 'a normal testcase request must normalize to its basename');
|
|
|
|
$judge_rows_text = "|Expected|Yours\n|--|--\n|a|b|student \t\n";
|
|
$judge_rows = llm_guidance_parse_diff_rows($judge_rows_text);
|
|
guidance_assert(count($judge_rows['expected']) === 1, 'the Markdown separator must not become a fake diff row');
|
|
guidance_assert($judge_rows['expected'][0] === 'a|b', 'pipes in expected output must stay on the expected side');
|
|
guidance_assert($judge_rows['yours'][0] === "student \t", 'trailing student whitespace must remain visible to PE comparison');
|
|
$judge_rows_attempt = llm_guidance_output_attempt($judge_rows_text);
|
|
guidance_assert($judge_rows_attempt['pairs'] === 1 && $judge_rows_attempt['expected_chars'] === 3, 'completion metrics must use the same final-pipe judge delimiter');
|
|
|
|
$full_diff_info = "========[3.out]=========\n\n------test in top 100 lines------\nsecret input\n\n------test out top 100 lines-----\nsecret answer\n\n------user out top 100 lines-----\nguess\n\n------diff out 200 lines-----\nsecret answer | guess\n\n==============================\n";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($full_diff_info), 'OJ_FULL_DIFF hidden sections must be detected');
|
|
guidance_assert(llm_guidance_diff_testcase_names($full_diff_info) === array('3'), 'full diff testcase header must be recognized without returning hidden values');
|
|
$truncated_full_diff = "========[3.out]=========\n\n------test in top 100 lines------\nsecret input\n\n3.out\n--\n|Expected|Yours\n|--|--\n|secret|guess\n";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($truncated_full_diff), 'a truncated full diff must fail closed after its first section marker');
|
|
$literal_newline_full_diff = "========[3.out]=========\\n------test in top 100 lines------\\nsecret input";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($literal_newline_full_diff), 'a shell-preserved literal-newline full marker must fail closed');
|
|
$tail_only_full_diff = "\\n------diff out 200 lines-----\\nsecret answer | guess";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($tail_only_full_diff), 'a tail-only full diff marker must also fail closed');
|
|
|
|
$garbage_output_info = "========[1.out]========\nExpected | Yours\n42 | 00\n========================\n";
|
|
$garbage_attempt = llm_guidance_output_attempt($garbage_output_info);
|
|
$garbage_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $garbage_attempt);
|
|
guidance_assert($garbage_non_oi['mode'] === 'focused_hint', 'equal-length unrelated output must not unlock diff');
|
|
|
|
$compact_wa_source = 'int main(){int n;cin>>n;cout<<(n+1);}';
|
|
$compact_wa = llm_guidance_assess($compact_wa_source, 6, 0);
|
|
$compact_near_wa = llm_guidance_refine_with_output_attempt($compact_wa, 6, $near_attempt);
|
|
guidance_assert($compact_near_wa['mode'] !== 'output_diff', 'compact source plus a near-looking output must not unlock diff');
|
|
|
|
$compact_high_wa = llm_guidance_assess('print(1)', 6, 0.85);
|
|
guidance_assert($compact_high_wa['mode'] === 'output_diff', 'compact WA with authoritative high pass rate must open diff');
|
|
|
|
$missing_diff = llm_guidance_refine_with_output_attempt($high_wa, 6, llm_guidance_output_attempt(''));
|
|
guidance_assert($missing_diff['mode'] === 'focused_hint' && !$missing_diff['show_output_diff'], 'high completion without a structured diff must use focused guidance');
|
|
|
|
$safe_focus = llm_guidance_render_focus_payload('{"category":"condition","line":7,"extra":"<img onerror=alert(1)>"}', $substantial);
|
|
guidance_assert(strpos($safe_focus, '第 7 行附近') !== false, 'allowlisted focus payload must preserve a valid student line');
|
|
guidance_assert(strpos($safe_focus, '<img') === false, 'free-form JSON fields must never reach rendered guidance');
|
|
guidance_assert(llm_guidance_render_focus_payload('需要排序加双指针', $substantial) === null, 'free-form algorithm prose must be rejected');
|
|
guidance_assert(llm_guidance_render_focus_payload('{"category":"solve","line":1}', $substantial) === null, 'non-allowlisted categories must be rejected');
|
|
|
|
$medium_re = llm_guidance_assess($substantial, 10, 0.35);
|
|
guidance_assert($medium_re['mode'] === 'focused_hint', 'nontrivial RE must receive a focused hint');
|
|
|
|
$blank_ce = llm_guidance_assess($blank, 11, 0);
|
|
guidance_assert($blank_ce['mode'] === 'flowchart', 'blank/template CE must not receive answer-shaped details');
|
|
|
|
$high_ce = llm_guidance_assess($substantial, 11, 0);
|
|
guidance_assert($high_ce['mode'] === 'compile_location', 'nontrivial CE must use location-only guidance');
|
|
|
|
$compile_error = "main.cpp:6:14: error: expected ';' before '}' token\n";
|
|
$locations = llm_guidance_extract_compile_locations($compile_error, $substantial, 1, 4);
|
|
guidance_assert(count($locations) === 1, 'GCC compile location must be parsed');
|
|
guidance_assert($locations[0]['line'] === 6 && $locations[0]['column'] === 14, 'compile line and column must be preserved');
|
|
guidance_assert(isset($locations[0]['excerpt'][0]['text']), 'compile location must contain student-source context');
|
|
guidance_assert(!isset($locations[0]['expected']) && !isset($locations[0]['replacement']), 'compile location must never contain an expected/replacement side');
|
|
|
|
$msvc_error = "main.cpp(7,3): error C2143: syntax error: missing ';' before '}'";
|
|
$msvc_locations = llm_guidance_extract_compile_locations($msvc_error, $substantial, 0, 4);
|
|
guidance_assert(count($msvc_locations) === 1, 'MSVC compile location must be parsed');
|
|
guidance_assert($msvc_locations[0]['line'] === 7 && $msvc_locations[0]['column'] === 3, 'MSVC line and column must be preserved');
|
|
|
|
$fpc_locations = llm_guidance_extract_compile_locations('main.pas(2,3) Error: Identifier not found', $substantial, 0, 4);
|
|
guidance_assert(count($fpc_locations) === 1 && $fpc_locations[0]['line'] === 2, 'FPC compile location must be parsed');
|
|
|
|
$go_locations = llm_guidance_extract_compile_locations('./main.go:3:5: undefined: value', $substantial, 0, 4);
|
|
guidance_assert(count($go_locations) === 1 && $go_locations[0]['column'] === 5, 'Go compile location must be parsed');
|
|
|
|
$rust_error = "error[E0425]: cannot find value `x` in this scope\n --> main.rs:4:9\n";
|
|
$rust_locations = llm_guidance_extract_compile_locations($rust_error, $substantial, 0, 4);
|
|
guidance_assert(count($rust_locations) === 1 && $rust_locations[0]['line'] === 4, 'Rust compile location must be parsed');
|
|
|
|
$php_locations = llm_guidance_extract_compile_locations('PHP Parse error: syntax error in Main.php on line 5', $substantial, 0, 4);
|
|
guidance_assert(count($php_locations) === 1 && $php_locations[0]['line'] === 5, 'PHP parse location must be parsed');
|
|
|
|
$bash_locations = llm_guidance_extract_compile_locations('Main.sh: line 6: syntax error near unexpected token', $substantial, 0, 4);
|
|
guidance_assert(count($bash_locations) === 1 && $bash_locations[0]['line'] === 6, 'Bash parse location must be parsed');
|
|
|
|
$ruby_locations = llm_guidance_extract_compile_locations('Main.rb:7: syntax error, unexpected end-of-input', $substantial, 0, 4);
|
|
guidance_assert(count($ruby_locations) === 1 && $ruby_locations[0]['line'] === 7, 'Ruby parse location must be parsed');
|
|
|
|
echo "llm_guidance_policy_test: OK" . PHP_EOL;
|