259 lines
18 KiB
PHP
259 lines
18 KiB
PHP
<?php
|
|
require_once __DIR__ . '/../web/include/llm_guidance.inc.php';
|
|
|
|
function guidance_assert($condition, $message) {
|
|
if (!$condition) {
|
|
fwrite(STDERR, "FAIL: " . $message . PHP_EOL);
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
$empty_source = '';
|
|
$main_only = "#include <bits/stdc++.h>\nusing namespace std;\nint main(){ return 0; }";
|
|
$substantial = <<<'CPP'
|
|
#include <bits/stdc++.h>
|
|
using namespace std;
|
|
int main() {
|
|
int n, answer = 0;
|
|
cin >> n;
|
|
for (int i = 0; i < n; ++i) {
|
|
int value;
|
|
cin >> value;
|
|
answer += value;
|
|
}
|
|
cout << answer << "\n";
|
|
return 0;
|
|
}
|
|
CPP;
|
|
|
|
$short_cpp = 'int main(){int n;cin>>n;int ans=n+1;cout<<ans;}';
|
|
$python_one_line = 'n=int(input());answer=n+1;print(answer)';
|
|
$javascript_short = <<<'JS'
|
|
const fs=require("fs");
|
|
const n=Number(fs.readFileSync(0,"utf8").trim());
|
|
const answer=n+1;
|
|
console.log(answer);
|
|
JS;
|
|
$pascal_short = <<<'PASCAL'
|
|
var n, answer: longint;
|
|
begin
|
|
readln(n);
|
|
answer := n + 1;
|
|
writeln(answer);
|
|
end.
|
|
PASCAL;
|
|
$csharp_short = 'var n=int.Parse(Console.ReadLine()); var answer=n+1; Console.WriteLine(answer);';
|
|
$go_short = 'func main(){var n int; fmt.Scan(&n); answer:=n+1; fmt.Println(answer)}';
|
|
$rust_short = 'let mut s=String::new(); std::io::stdin().read_line(&mut s); let n:i32=s.trim().parse().unwrap(); println!("{}",n+1);';
|
|
$lua_short = 'local n=io.read("*n"); local answer=n+1; print(answer)';
|
|
$bash_short = 'read n; answer=$((n+1)); echo "$answer"';
|
|
$comparison_short = 'n=int(input());print(n<0)';
|
|
|
|
guidance_assert(llm_guidance_assess($substantial, 4, 1)['mode'] === 'hidden', 'AC must hide guidance');
|
|
guidance_assert(llm_guidance_assess($substantial, 3, 0)['mode'] === 'hidden', 'pending must hide guidance');
|
|
guidance_assert(llm_guidance_assess($substantial, 13, 0)['mode'] === 'hidden', 'test run must hide guidance');
|
|
|
|
$comment_only = "// for if while cin cout answer = 42\n/* int main() { cout << answer; } */";
|
|
$hash_comment_only = "# for if while input print answer = 1\n# if answer: print(answer)\n# more comments\n# still no code";
|
|
$constant_output = 'int main(){cout<<42;}';
|
|
$low_wa_sources = array(
|
|
'empty source' => $empty_source,
|
|
'C/C++ comment-only source' => $comment_only,
|
|
'hash comment-only source' => $hash_comment_only,
|
|
'main-only template' => $main_only,
|
|
'constant-output source' => $constant_output,
|
|
);
|
|
foreach ($low_wa_sources as $label => $source) {
|
|
$assessment = llm_guidance_assess($source, 6, 0);
|
|
guidance_assert($assessment['level'] === 'low', $label . ' WA must stay low completion');
|
|
guidance_assert($assessment['mode'] === 'flowchart', $label . ' WA must receive flowchart guidance');
|
|
guidance_assert($assessment['show_output_diff'] === false, $label . ' WA must not auto-open output diff');
|
|
}
|
|
|
|
$completed_wa_sources = array(
|
|
'short C++ input/process/output' => $short_cpp,
|
|
'Python one-line input/process/output' => $python_one_line,
|
|
'JavaScript input/process/output' => $javascript_short,
|
|
'Pascal input/process/output' => $pascal_short,
|
|
'C# input/process/output' => $csharp_short,
|
|
'Go input/process/output' => $go_short,
|
|
'Rust input/process/output' => $rust_short,
|
|
'Lua input/process/output' => $lua_short,
|
|
'Bash input/process/output' => $bash_short,
|
|
'comparison input/process/output' => $comparison_short,
|
|
);
|
|
foreach ($completed_wa_sources as $label => $source) {
|
|
$assessment = llm_guidance_assess($source, 6, 0);
|
|
guidance_assert($assessment['level'] === 'high', $label . ' must be high completion even when pass_rate=0');
|
|
guidance_assert($assessment['mode'] === 'output_diff', $label . ' must open output diff even when pass_rate=0');
|
|
guidance_assert($assessment['show_output_diff'] === true, $label . ' must expose the existing judge diff');
|
|
}
|
|
|
|
$high_wa = llm_guidance_assess($substantial, 6, 0.85);
|
|
guidance_assert($high_wa['mode'] === 'output_diff', 'WA with a high judged pass rate must open output diff');
|
|
guidance_assert($high_wa['show_output_diff'] === true, 'high-completion WA must expose output diff');
|
|
|
|
$non_oi_wa = llm_guidance_assess($substantial, 6, 0);
|
|
guidance_assert($non_oi_wa['level'] === 'high', 'a complete input/process/output attempt must be high when pass_rate=0');
|
|
guidance_assert($non_oi_wa['mode'] === 'output_diff', 'pass_rate=0 must not hide the existing judge diff for a complete attempt');
|
|
|
|
$high_pe = llm_guidance_assess($substantial, 5, 0);
|
|
guidance_assert($high_pe['mode'] === 'output_diff', 'substantial PE must open output diff');
|
|
$compact_pe = llm_guidance_assess('print(1)', 5, 0);
|
|
guidance_assert($compact_pe['mode'] === 'output_diff', 'judge-proven compact PE must open output diff');
|
|
|
|
$empty_output_info = "========[1.out]========\nExpected | Yours\n42 | \n========================\n";
|
|
$empty_attempt = llm_guidance_output_attempt($empty_output_info);
|
|
guidance_assert($empty_attempt['available'] && $empty_attempt['yours_chars'] === 0, 'empty judge output must be detected');
|
|
$complete_empty_case = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $empty_attempt);
|
|
guidance_assert($complete_empty_case['level'] === 'high', 'one empty failing output must not erase complete source evidence');
|
|
guidance_assert($complete_empty_case['mode'] === 'output_diff', 'one empty failing output must not downgrade complete source to flowchart');
|
|
guidance_assert($complete_empty_case['show_output_diff'] === true, 'one empty failing output must preserve the existing judge diff');
|
|
|
|
$trusted_empty_case = llm_guidance_refine_with_output_attempt($high_wa, 6, $empty_attempt);
|
|
guidance_assert($trusted_empty_case['mode'] === 'output_diff', 'one empty failing case must not erase authoritative overall pass progress');
|
|
|
|
$near_output_info = "========[1.out]========\nExpected | Yours\n12345 | 12346\n========================\n";
|
|
$near_attempt = llm_guidance_output_attempt($near_output_info);
|
|
$near_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $near_attempt);
|
|
guidance_assert($near_non_oi['mode'] === 'output_diff', 'output-shape heuristics must not demote a complete source attempt');
|
|
|
|
$oracle_output_info = "========[1.out]========\nExpected | Yours\n1000000000 | 0000000000\n========================\n";
|
|
$oracle_attempt = llm_guidance_output_attempt($oracle_output_info);
|
|
$oracle_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $oracle_attempt);
|
|
guidance_assert($oracle_non_oi['mode'] === 'output_diff', 'output contents must not override complete-source classification');
|
|
|
|
$two_case_info = "1.out\n--\n|Expected|Yours\n|--|--\n|1|0\n\n2.out\n--\n|Expected|Yours\n|--|--\n|2|3\n";
|
|
$two_case_names = llm_guidance_diff_testcase_names($two_case_info);
|
|
guidance_assert($two_case_names === array('1', '2'), 'only judge-recorded simple diff testcase names may be authorized');
|
|
guidance_assert(llm_guidance_testcase_base('../2.out') === null, 'testcase paths and traversal must be rejected');
|
|
guidance_assert(llm_guidance_testcase_base('2.out') === '2', 'a normal testcase request must normalize to its basename');
|
|
|
|
$judge_rows_text = "|Expected|Yours\n|--|--\n|a|b|student \t\n";
|
|
$judge_rows = llm_guidance_parse_diff_rows($judge_rows_text);
|
|
guidance_assert(count($judge_rows['expected']) === 1, 'the Markdown separator must not become a fake diff row');
|
|
guidance_assert($judge_rows['expected'][0] === 'a|b', 'pipes in expected output must stay on the expected side');
|
|
guidance_assert($judge_rows['yours'][0] === "student \t", 'trailing student whitespace must remain visible to PE comparison');
|
|
$judge_rows_attempt = llm_guidance_output_attempt($judge_rows_text);
|
|
guidance_assert($judge_rows_attempt['pairs'] === 1 && $judge_rows_attempt['expected_chars'] === 3, 'completion metrics must use the same final-pipe judge delimiter');
|
|
|
|
$legacy_backslash_info = "========[legacy.out]=========\n"
|
|
. "Expected | Yours\n"
|
|
. "42 \\ 41\n\n"
|
|
. "==============================\n";
|
|
$legacy_blocks = llm_guidance_parse_diff_blocks($legacy_backslash_info);
|
|
guidance_assert(count($legacy_blocks) === 1, 'legacy blocks with a blank line before the closing delimiter must parse');
|
|
guidance_assert($legacy_blocks[0]['expected'] === array('42'), 'legacy backslash diff must preserve expected rows');
|
|
guidance_assert($legacy_blocks[0]['yours'] === array('41'), 'legacy backslash diff must preserve student rows');
|
|
$legacy_attempt = llm_guidance_output_attempt($legacy_backslash_info);
|
|
guidance_assert($legacy_attempt['available'] && $legacy_attempt['pairs'] === 1, 'legacy backslash rows must count as an output attempt');
|
|
|
|
$full_diff_info = "========[3.out]=========\n\n------test in top 100 lines------\nsecret input\n\n------test out top 100 lines-----\nsecret answer\n\n------user out top 100 lines-----\nguess\n\n------diff out 200 lines-----\nsecret answer | guess\n\n==============================\n";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($full_diff_info), 'OJ_FULL_DIFF hidden sections must be detected');
|
|
guidance_assert(llm_guidance_diff_testcase_names($full_diff_info) === array('3'), 'full diff testcase header must be recognized without returning hidden values');
|
|
$truncated_full_diff = "========[3.out]=========\n\n------test in top 100 lines------\nsecret input\n\n3.out\n--\n|Expected|Yours\n|--|--\n|secret|guess\n";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($truncated_full_diff), 'a truncated full diff must fail closed after its first section marker');
|
|
$literal_newline_full_diff = "========[3.out]=========\\n------test in top 100 lines------\\nsecret input";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($literal_newline_full_diff), 'a shell-preserved literal-newline full marker must fail closed');
|
|
$tail_only_full_diff = "\\n------diff out 200 lines-----\\nsecret answer | guess";
|
|
guidance_assert(llm_guidance_has_full_diff_sections($tail_only_full_diff), 'a tail-only full diff marker must also fail closed');
|
|
$redacted_full_diff = llm_guidance_redact_full_diff($full_diff_info);
|
|
guidance_assert(strpos($redacted_full_diff, 'secret input') === false, 'full diff redaction must remove hidden testcase input');
|
|
guidance_assert(strpos($redacted_full_diff, 'secret answer') === false, 'full diff redaction must remove expected output and combined diff');
|
|
guidance_assert(strpos($redacted_full_diff, 'guess') === false, 'full diff redaction must not parse attacker-controlled section markers');
|
|
guidance_assert(strpos($redacted_full_diff, '当前账号权限省略') !== false, 'full diff redaction must explain omitted judge data');
|
|
$redacted_literal_tail = llm_guidance_redact_full_diff($tail_only_full_diff);
|
|
guidance_assert(strpos($redacted_literal_tail, 'secret answer') === false, 'literal-newline full diff tails must also redact protected data');
|
|
|
|
$garbage_output_info = "========[1.out]========\nExpected | Yours\n42 | 00\n========================\n";
|
|
$garbage_attempt = llm_guidance_output_attempt($garbage_output_info);
|
|
$garbage_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $garbage_attempt);
|
|
guidance_assert($garbage_non_oi['mode'] === 'output_diff', 'unrelated output must not demote a complete source attempt');
|
|
|
|
$missing_diff = llm_guidance_refine_with_output_attempt($high_wa, 6, llm_guidance_output_attempt(''));
|
|
guidance_assert($missing_diff['mode'] === 'judge_output' && !$missing_diff['show_output_diff'], 'high completion without a structured diff must use deterministic judge-output guidance');
|
|
|
|
$low_confidence_focus = llm_guidance_render_focus_payload(
|
|
'{"category":"condition","line":7,"confidence":"low"}',
|
|
$substantial
|
|
);
|
|
guidance_assert($low_confidence_focus !== null, 'allowlisted low-confidence focus payload must render');
|
|
guidance_assert(strpos($low_confidence_focus, '第 7 行') === false, 'low-confidence focus must not display a source line');
|
|
|
|
$medium_confidence_focus = llm_guidance_render_focus_payload(
|
|
'{"category":"condition","line":7,"confidence":"medium"}',
|
|
$substantial
|
|
);
|
|
guidance_assert($medium_confidence_focus !== null, 'allowlisted medium-confidence focus payload must render');
|
|
guidance_assert(strpos($medium_confidence_focus, '第 7 行') === false, 'only high-confidence focus may display a source line');
|
|
|
|
$high_confidence_focus = llm_guidance_render_focus_payload(
|
|
'{"category":"condition","line":7,"confidence":"high","extra":"<img onerror=alert(1)>"}',
|
|
$substantial
|
|
);
|
|
guidance_assert(strpos($high_confidence_focus, '第 7 行附近') !== false, 'high-confidence focus must preserve a valid student line');
|
|
guidance_assert(strpos($high_confidence_focus, '<img') === false, 'free-form JSON fields must never reach rendered guidance');
|
|
guidance_assert(llm_guidance_render_focus_payload('需要排序加双指针', $substantial) === null, 'free-form algorithm prose must be rejected');
|
|
guidance_assert(llm_guidance_render_focus_payload('{"category":"solve","line":1}', $substantial) === null, 'non-allowlisted categories must be rejected');
|
|
guidance_assert(llm_guidance_render_focus_payload('{"category":"condition","line":7}', $substantial) === null, 'missing confidence must be rejected');
|
|
guidance_assert(llm_guidance_render_focus_payload('{"category":"condition","line":7,"confidence":"certain"}', $substantial) === null, 'non-allowlisted confidence must be rejected');
|
|
|
|
$medium_re = llm_guidance_assess($substantial, 10, 0.35);
|
|
guidance_assert($medium_re['mode'] === 'focused_hint', 'nontrivial RE must receive a focused hint');
|
|
|
|
$short_ce = llm_guidance_assess('print(', 11, 0);
|
|
guidance_assert($short_ce['mode'] === 'compile_location', 'short CE source must still use compiler location guidance');
|
|
|
|
$high_ce = llm_guidance_assess($substantial, 11, 0);
|
|
guidance_assert($high_ce['mode'] === 'compile_location', 'nontrivial CE must use location-only guidance');
|
|
|
|
$compile_error = "main.cpp:6:14: error: expected ';' before '}' token\n";
|
|
$locations = llm_guidance_extract_compile_locations($compile_error, $substantial, 1, 4);
|
|
guidance_assert(count($locations) === 1, 'GCC compile location must be parsed');
|
|
guidance_assert($locations[0]['line'] === 6 && $locations[0]['column'] === 14, 'compile line and column must be preserved');
|
|
guidance_assert(isset($locations[0]['excerpt'][0]['text']), 'compile location must contain student-source context');
|
|
guidance_assert(!isset($locations[0]['expected']) && !isset($locations[0]['replacement']), 'compile location must never contain an expected/replacement side');
|
|
|
|
$msvc_error = "main.cpp(7,3): error C2143: syntax error: missing ';' before '}'";
|
|
$msvc_locations = llm_guidance_extract_compile_locations($msvc_error, $substantial, 0, 4);
|
|
guidance_assert(count($msvc_locations) === 1, 'MSVC compile location must be parsed');
|
|
guidance_assert($msvc_locations[0]['line'] === 7 && $msvc_locations[0]['column'] === 3, 'MSVC line and column must be preserved');
|
|
|
|
$fpc_locations = llm_guidance_extract_compile_locations('main.pas(2,3) Error: Identifier not found', $substantial, 0, 4);
|
|
guidance_assert(count($fpc_locations) === 1 && $fpc_locations[0]['line'] === 2, 'FPC compile location must be parsed');
|
|
|
|
$go_locations = llm_guidance_extract_compile_locations('./main.go:3:5: undefined: value', $substantial, 0, 4);
|
|
guidance_assert(count($go_locations) === 1 && $go_locations[0]['column'] === 5, 'Go compile location must be parsed');
|
|
|
|
$rust_error = "error[E0425]: cannot find value `x` in this scope\n --> main.rs:4:9\n";
|
|
$rust_locations = llm_guidance_extract_compile_locations($rust_error, $substantial, 0, 4);
|
|
guidance_assert(count($rust_locations) === 1 && $rust_locations[0]['line'] === 4, 'Rust compile location must be parsed');
|
|
|
|
$php_locations = llm_guidance_extract_compile_locations('PHP Parse error: syntax error in Main.php on line 5', $substantial, 0, 4);
|
|
guidance_assert(count($php_locations) === 1 && $php_locations[0]['line'] === 5, 'PHP parse location must be parsed');
|
|
|
|
$bash_locations = llm_guidance_extract_compile_locations('Main.sh: line 6: syntax error near unexpected token', $substantial, 0, 4);
|
|
guidance_assert(count($bash_locations) === 1 && $bash_locations[0]['line'] === 6, 'Bash parse location must be parsed');
|
|
|
|
$ruby_locations = llm_guidance_extract_compile_locations('Main.rb:7: syntax error, unexpected end-of-input', $substantial, 0, 4);
|
|
guidance_assert(count($ruby_locations) === 1 && $ruby_locations[0]['line'] === 7, 'Ruby parse location must be parsed');
|
|
|
|
$reinfo_source = file_get_contents(__DIR__ . '/../web/reinfo.php');
|
|
guidance_assert(strpos($reinfo_source, '当前提交完成度不足') === false, 'reinfo must not replace judge output with an AI completion warning');
|
|
guidance_assert(strpos($reinfo_source, '$diff_blocks = array();') === false, 'AI policy must not clear parsed judge diff blocks');
|
|
guidance_assert(strpos($reinfo_source, 'llm_guidance_redact_full_diff') !== false, 'ordinary full-diff output must redact protected testcase data');
|
|
|
|
$ceinfo_source = file_get_contents(__DIR__ . '/../web/ceinfo.php');
|
|
guidance_assert(strpos($ceinfo_source, 'llm_guidance_extract_compile_locations') !== false, 'CE route must expose compiler-owned source locations');
|
|
|
|
$download_source = file_get_contents(__DIR__ . '/../web/download.php');
|
|
guidance_assert(strpos($download_source, 'llm_guidance_assess(') === false, 'testcase download authorization must not depend on AI completion');
|
|
|
|
$review_endpoint_source = file_get_contents(__DIR__ . '/../web/llm-review.php');
|
|
guidance_assert(strpos($review_endpoint_source, 'llm_review_fallback_cache_marker') !== false, 'invalid model output must use a distinct bounded fallback cache');
|
|
guidance_assert(strpos($review_endpoint_source, 'llm_review_cached_fallback') !== false, 'fallback cache must have an explicit TTL reader');
|
|
guidance_assert(strpos($review_endpoint_source, 'llm_review_session_limits') !== false, 'missing rate-limit migration must have a bounded session fallback');
|
|
guidance_assert(strpos($review_endpoint_source, 'GET_LOCK') !== false, 'missing rate-limit migration must prevent cross-session concurrent model calls');
|
|
|
|
echo "llm_guidance_policy_test: OK" . PHP_EOL;
|