klarkxy/hustoj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): relax guidance and preserve judge output

Browse Source
This commit is contained in:
klarkxy
2026-07-24 14:06:51 +08:00
parent 44812f70e3
commit fd53bab639
7 changed files with 548 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/llm_guidance_policy_test.php
+127 -29
View File
@@ -8,7 +8,8 @@ function guidance_assert($condition, $message) {
}
}
$blank = "#include <bits/stdc++.h>\nusing namespace std;\nint main(){ return 0; }";
$empty_source = '';
$main_only = "#include <bits/stdc++.h>\nusing namespace std;\nint main(){ return 0; }";
$substantial = <<<'CPP'
#include <bits/stdc++.h>
using namespace std;
@@ -25,26 +26,76 @@ int main() {
}
CPP;
$short_cpp = 'int main(){int n;cin>>n;int ans=n+1;cout<<ans;}';
$python_one_line = 'n=int(input());answer=n+1;print(answer)';
$javascript_short = <<<'JS'
const fs=require("fs");
const n=Number(fs.readFileSync(0,"utf8").trim());
const answer=n+1;
console.log(answer);
JS;
$pascal_short = <<<'PASCAL'
var n, answer: longint;
begin
readln(n);
answer := n + 1;
writeln(answer);
end.
PASCAL;
$csharp_short = 'var n=int.Parse(Console.ReadLine()); var answer=n+1; Console.WriteLine(answer);';
$go_short = 'func main(){var n int; fmt.Scan(&n); answer:=n+1; fmt.Println(answer)}';
$rust_short = 'let mut s=String::new(); std::io::stdin().read_line(&mut s); let n:i32=s.trim().parse().unwrap(); println!("{}",n+1);';
$lua_short = 'local n=io.read("*n"); local answer=n+1; print(answer)';
$bash_short = 'read n; answer=$((n+1)); echo "$answer"';
$comparison_short = 'n=int(input());print(n<0)';
guidance_assert(llm_guidance_assess($substantial, 4, 1)['mode'] === 'hidden', 'AC must hide guidance');
guidance_assert(llm_guidance_assess($substantial, 3, 0)['mode'] === 'hidden', 'pending must hide guidance');
guidance_assert(llm_guidance_assess($substantial, 13, 0)['mode'] === 'hidden', 'test run must hide guidance');
$blank_wa = llm_guidance_assess($blank, 6, 0);
guidance_assert($blank_wa['mode'] === 'flowchart', 'blank/template WA must receive flowchart guidance');
guidance_assert($blank_wa['show_output_diff'] === false, 'blank/template WA must not expose expected-output diff');
$comment_only = "// for if while cin cout answer = 42\n/* int main() { cout << answer; } */";
guidance_assert(llm_guidance_assess($comment_only, 6, 0)['mode'] === 'flowchart', 'comment-only submissions must stay low completion');
$hash_comment_only = "# for if while input print answer = 1\n# if answer: print(answer)\n# more comments\n# still no code";
guidance_assert(llm_guidance_assess($hash_comment_only, 6, 0)['mode'] === 'flowchart', '# comment-only submissions must stay low completion');
$constant_output = 'int main(){cout<<42;}';
$low_wa_sources = array(
'empty source' => $empty_source,
'C/C++ comment-only source' => $comment_only,
'hash comment-only source' => $hash_comment_only,
'main-only template' => $main_only,
'constant-output source' => $constant_output,
);
foreach ($low_wa_sources as $label => $source) {
$assessment = llm_guidance_assess($source, 6, 0);
guidance_assert($assessment['level'] === 'low', $label . ' WA must stay low completion');
guidance_assert($assessment['mode'] === 'flowchart', $label . ' WA must receive flowchart guidance');
guidance_assert($assessment['show_output_diff'] === false, $label . ' WA must not auto-open output diff');
}
$completed_wa_sources = array(
'short C++ input/process/output' => $short_cpp,
'Python one-line input/process/output' => $python_one_line,
'JavaScript input/process/output' => $javascript_short,
'Pascal input/process/output' => $pascal_short,
'C# input/process/output' => $csharp_short,
'Go input/process/output' => $go_short,
'Rust input/process/output' => $rust_short,
'Lua input/process/output' => $lua_short,
'Bash input/process/output' => $bash_short,
'comparison input/process/output' => $comparison_short,
);
foreach ($completed_wa_sources as $label => $source) {
$assessment = llm_guidance_assess($source, 6, 0);
guidance_assert($assessment['level'] === 'high', $label . ' must be high completion even when pass_rate=0');
guidance_assert($assessment['mode'] === 'output_diff', $label . ' must open output diff even when pass_rate=0');
guidance_assert($assessment['show_output_diff'] === true, $label . ' must expose the existing judge diff');
}
$high_wa = llm_guidance_assess($substantial, 6, 0.85);
guidance_assert($high_wa['mode'] === 'output_diff', 'WA with a high judged pass rate must open output diff');
guidance_assert($high_wa['show_output_diff'] === true, 'high-completion WA must expose output diff');
$non_oi_wa = llm_guidance_assess($substantial, 6, 0);
guidance_assert($non_oi_wa['mode'] === 'focused_hint', 'pass_rate=0 must not unlock hidden output from source shape alone');
guidance_assert($non_oi_wa['level'] === 'high', 'a complete input/process/output attempt must be high when pass_rate=0');
guidance_assert($non_oi_wa['mode'] === 'output_diff', 'pass_rate=0 must not hide the existing judge diff for a complete attempt');
$high_pe = llm_guidance_assess($substantial, 5, 0);
guidance_assert($high_pe['mode'] === 'output_diff', 'substantial PE must open output diff');
@@ -54,9 +105,10 @@ guidance_assert($compact_pe['mode'] === 'output_diff', 'judge-proven compact PE
$empty_output_info = "========[1.out]========\nExpected | Yours\n42 | \n========================\n";
$empty_attempt = llm_guidance_output_attempt($empty_output_info);
guidance_assert($empty_attempt['available'] && $empty_attempt['yours_chars'] === 0, 'empty judge output must be detected');
$padded_blank = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $empty_attempt);
guidance_assert($padded_blank['mode'] === 'flowchart', 'long source with empty output must be downgraded to flowchart');
guidance_assert($padded_blank['show_output_diff'] === false, 'long source with empty output must not expose expected output');
$complete_empty_case = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $empty_attempt);
guidance_assert($complete_empty_case['level'] === 'high', 'one empty failing output must not erase complete source evidence');
guidance_assert($complete_empty_case['mode'] === 'output_diff', 'one empty failing output must not downgrade complete source to flowchart');
guidance_assert($complete_empty_case['show_output_diff'] === true, 'one empty failing output must preserve the existing judge diff');
$trusted_empty_case = llm_guidance_refine_with_output_attempt($high_wa, 6, $empty_attempt);
guidance_assert($trusted_empty_case['mode'] === 'output_diff', 'one empty failing case must not erase authoritative overall pass progress');
@@ -64,12 +116,12 @@ guidance_assert($trusted_empty_case['mode'] === 'output_diff', 'one empty failin
$near_output_info = "========[1.out]========\nExpected | Yours\n12345 | 12346\n========================\n";
$near_attempt = llm_guidance_output_attempt($near_output_info);
$near_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $near_attempt);
guidance_assert($near_non_oi['mode'] === 'focused_hint', 'a near-looking hidden output must not unlock diff');
guidance_assert($near_non_oi['mode'] === 'output_diff', 'output-shape heuristics must not demote a complete source attempt');
$oracle_output_info = "========[1.out]========\nExpected | Yours\n1000000000 | 0000000000\n========================\n";
$oracle_attempt = llm_guidance_output_attempt($oracle_output_info);
$oracle_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $oracle_attempt);
guidance_assert($oracle_non_oi['mode'] === 'focused_hint', 'a repeated-output guess must not unlock hidden expected output');
guidance_assert($oracle_non_oi['mode'] === 'output_diff', 'output contents must not override complete-source classification');
$two_case_info = "1.out\n--\n|Expected|Yours\n|--|--\n|1|0\n\n2.out\n--\n|Expected|Yours\n|--|--\n|2|3\n";
$two_case_names = llm_guidance_diff_testcase_names($two_case_info);
@@ -85,6 +137,17 @@ guidance_assert($judge_rows['yours'][0] === "student \t", 'trailing student whi
$judge_rows_attempt = llm_guidance_output_attempt($judge_rows_text);
guidance_assert($judge_rows_attempt['pairs'] === 1 && $judge_rows_attempt['expected_chars'] === 3, 'completion metrics must use the same final-pipe judge delimiter');
$legacy_backslash_info = "========[legacy.out]=========\n"
. "Expected | Yours\n"
. "42 \\ 41\n\n"
. "==============================\n";
$legacy_blocks = llm_guidance_parse_diff_blocks($legacy_backslash_info);
guidance_assert(count($legacy_blocks) === 1, 'legacy blocks with a blank line before the closing delimiter must parse');
guidance_assert($legacy_blocks[0]['expected'] === array('42'), 'legacy backslash diff must preserve expected rows');
guidance_assert($legacy_blocks[0]['yours'] === array('41'), 'legacy backslash diff must preserve student rows');
$legacy_attempt = llm_guidance_output_attempt($legacy_backslash_info);
guidance_assert($legacy_attempt['available'] && $legacy_attempt['pairs'] === 1, 'legacy backslash rows must count as an output attempt');
$full_diff_info = "========[3.out]=========\n\n------test in top 100 lines------\nsecret input\n\n------test out top 100 lines-----\nsecret answer\n\n------user out top 100 lines-----\nguess\n\n------diff out 200 lines-----\nsecret answer | guess\n\n==============================\n";
guidance_assert(llm_guidance_has_full_diff_sections($full_diff_info), 'OJ_FULL_DIFF hidden sections must be detected');
guidance_assert(llm_guidance_diff_testcase_names($full_diff_info) === array('3'), 'full diff testcase header must be recognized without returning hidden values');
@@ -94,34 +157,52 @@ $literal_newline_full_diff = "========[3.out]=========\\n------test in top 100 l
guidance_assert(llm_guidance_has_full_diff_sections($literal_newline_full_diff), 'a shell-preserved literal-newline full marker must fail closed');
$tail_only_full_diff = "\\n------diff out 200 lines-----\\nsecret answer | guess";
guidance_assert(llm_guidance_has_full_diff_sections($tail_only_full_diff), 'a tail-only full diff marker must also fail closed');
$redacted_full_diff = llm_guidance_redact_full_diff($full_diff_info);
guidance_assert(strpos($redacted_full_diff, 'secret input') === false, 'full diff redaction must remove hidden testcase input');
guidance_assert(strpos($redacted_full_diff, 'secret answer') === false, 'full diff redaction must remove expected output and combined diff');
guidance_assert(strpos($redacted_full_diff, 'guess') === false, 'full diff redaction must not parse attacker-controlled section markers');
guidance_assert(strpos($redacted_full_diff, '当前账号权限省略') !== false, 'full diff redaction must explain omitted judge data');
$redacted_literal_tail = llm_guidance_redact_full_diff($tail_only_full_diff);
guidance_assert(strpos($redacted_literal_tail, 'secret answer') === false, 'literal-newline full diff tails must also redact protected data');
$garbage_output_info = "========[1.out]========\nExpected | Yours\n42 | 00\n========================\n";
$garbage_attempt = llm_guidance_output_attempt($garbage_output_info);
$garbage_non_oi = llm_guidance_refine_with_output_attempt($non_oi_wa, 6, $garbage_attempt);
guidance_assert($garbage_non_oi['mode'] === 'focused_hint', 'equal-length unrelated output must not unlock diff');
$compact_wa_source = 'int main(){int n;cin>>n;cout<<(n+1);}';
$compact_wa = llm_guidance_assess($compact_wa_source, 6, 0);
$compact_near_wa = llm_guidance_refine_with_output_attempt($compact_wa, 6, $near_attempt);
guidance_assert($compact_near_wa['mode'] !== 'output_diff', 'compact source plus a near-looking output must not unlock diff');
$compact_high_wa = llm_guidance_assess('print(1)', 6, 0.85);
guidance_assert($compact_high_wa['mode'] === 'output_diff', 'compact WA with authoritative high pass rate must open diff');
guidance_assert($garbage_non_oi['mode'] === 'output_diff', 'unrelated output must not demote a complete source attempt');
$missing_diff = llm_guidance_refine_with_output_attempt($high_wa, 6, llm_guidance_output_attempt(''));
guidance_assert($missing_diff['mode'] === 'focused_hint' && !$missing_diff['show_output_diff'], 'high completion without a structured diff must use focused guidance');
guidance_assert($missing_diff['mode'] === 'judge_output' && !$missing_diff['show_output_diff'], 'high completion without a structured diff must use deterministic judge-output guidance');
$safe_focus = llm_guidance_render_focus_payload('{"category":"condition","line":7,"extra":"<img onerror=alert(1)>"}', $substantial);
guidance_assert(strpos($safe_focus, '第 7 行附近') !== false, 'allowlisted focus payload must preserve a valid student line');
guidance_assert(strpos($safe_focus, '<img') === false, 'free-form JSON fields must never reach rendered guidance');
$low_confidence_focus = llm_guidance_render_focus_payload(
'{"category":"condition","line":7,"confidence":"low"}',
$substantial
);
guidance_assert($low_confidence_focus !== null, 'allowlisted low-confidence focus payload must render');
guidance_assert(strpos($low_confidence_focus, '第 7 行') === false, 'low-confidence focus must not display a source line');
$medium_confidence_focus = llm_guidance_render_focus_payload(
'{"category":"condition","line":7,"confidence":"medium"}',
$substantial
);
guidance_assert($medium_confidence_focus !== null, 'allowlisted medium-confidence focus payload must render');
guidance_assert(strpos($medium_confidence_focus, '第 7 行') === false, 'only high-confidence focus may display a source line');
$high_confidence_focus = llm_guidance_render_focus_payload(
'{"category":"condition","line":7,"confidence":"high","extra":"<img onerror=alert(1)>"}',
$substantial
);
guidance_assert(strpos($high_confidence_focus, '第 7 行附近') !== false, 'high-confidence focus must preserve a valid student line');
guidance_assert(strpos($high_confidence_focus, '<img') === false, 'free-form JSON fields must never reach rendered guidance');
guidance_assert(llm_guidance_render_focus_payload('需要排序加双指针', $substantial) === null, 'free-form algorithm prose must be rejected');
guidance_assert(llm_guidance_render_focus_payload('{"category":"solve","line":1}', $substantial) === null, 'non-allowlisted categories must be rejected');
guidance_assert(llm_guidance_render_focus_payload('{"category":"condition","line":7}', $substantial) === null, 'missing confidence must be rejected');
guidance_assert(llm_guidance_render_focus_payload('{"category":"condition","line":7,"confidence":"certain"}', $substantial) === null, 'non-allowlisted confidence must be rejected');
$medium_re = llm_guidance_assess($substantial, 10, 0.35);
guidance_assert($medium_re['mode'] === 'focused_hint', 'nontrivial RE must receive a focused hint');
$blank_ce = llm_guidance_assess($blank, 11, 0);
guidance_assert($blank_ce['mode'] === 'flowchart', 'blank/template CE must not receive answer-shaped details');
$short_ce = llm_guidance_assess('print(', 11, 0);
guidance_assert($short_ce['mode'] === 'compile_location', 'short CE source must still use compiler location guidance');
$high_ce = llm_guidance_assess($substantial, 11, 0);
guidance_assert($high_ce['mode'] === 'compile_location', 'nontrivial CE must use location-only guidance');
@@ -157,4 +238,21 @@ guidance_assert(count($bash_locations) === 1 && $bash_locations[0]['line'] === 6
$ruby_locations = llm_guidance_extract_compile_locations('Main.rb:7: syntax error, unexpected end-of-input', $substantial, 0, 4);
guidance_assert(count($ruby_locations) === 1 && $ruby_locations[0]['line'] === 7, 'Ruby parse location must be parsed');
$reinfo_source = file_get_contents(__DIR__ . '/../web/reinfo.php');
guidance_assert(strpos($reinfo_source, '当前提交完成度不足') === false, 'reinfo must not replace judge output with an AI completion warning');
guidance_assert(strpos($reinfo_source, '$diff_blocks = array();') === false, 'AI policy must not clear parsed judge diff blocks');
guidance_assert(strpos($reinfo_source, 'llm_guidance_redact_full_diff') !== false, 'ordinary full-diff output must redact protected testcase data');
$ceinfo_source = file_get_contents(__DIR__ . '/../web/ceinfo.php');
guidance_assert(strpos($ceinfo_source, 'llm_guidance_extract_compile_locations') !== false, 'CE route must expose compiler-owned source locations');
$download_source = file_get_contents(__DIR__ . '/../web/download.php');
guidance_assert(strpos($download_source, 'llm_guidance_assess(') === false, 'testcase download authorization must not depend on AI completion');
$review_endpoint_source = file_get_contents(__DIR__ . '/../web/llm-review.php');
guidance_assert(strpos($review_endpoint_source, 'llm_review_fallback_cache_marker') !== false, 'invalid model output must use a distinct bounded fallback cache');
guidance_assert(strpos($review_endpoint_source, 'llm_review_cached_fallback') !== false, 'fallback cache must have an explicit TTL reader');
guidance_assert(strpos($review_endpoint_source, 'llm_review_session_limits') !== false, 'missing rate-limit migration must have a bounded session fallback');
guidance_assert(strpos($review_endpoint_source, 'GET_LOCK') !== false, 'missing rate-limit migration must prevent cross-session concurrent model calls');
echo "llm_guidance_policy_test: OK" . PHP_EOL;