klarkxy/hustoj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): relax guidance and preserve judge output

Browse Source
This commit is contained in:
klarkxy
2026-07-24 14:06:51 +08:00
parent 44812f70e3
commit fd53bab639
7 changed files with 548 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/include/llm_guidance.inc.php
+161 -81
View File
@@ -20,8 +20,6 @@ if (!function_exists('llm_guidance_source_metrics')) {
$clean = preg_replace('/\/\/[^\n]*/', ' ', $clean);
if ($clean === null) $clean = $source;
$non_space = preg_replace('/\s+/', '', $clean);
$non_space_chars = strlen($non_space === null ? '' : $non_space);
$lines = explode("\n", $clean);
$meaningful_lines = 0;
$activity_lines = 0;
@@ -35,6 +33,7 @@ if (!function_exists('llm_guidance_source_metrics')) {
// from inflating completion metrics.
if (strpos($line, '#') === 0) continue;
if (preg_match('/^using\s+namespace\b.*;$/i', $line)) continue;
if (preg_match('/^(?:package|import)\s+[\w.*]+;?$/i', $line)) continue;
if (preg_match('/^[{};]+$/', $line)) continue;
$meaningful_lines++;
@@ -49,25 +48,51 @@ if (!function_exists('llm_guidance_source_metrics')) {
}
$metric_source = implode("\n", $metric_lines);
$non_space = preg_replace('/\s+/', '', $metric_source);
$non_space_chars = strlen($non_space === null ? '' : $non_space);
$analysis_source = preg_replace('/"(?:\\\\.|[^"\\\\])*"/s', '""', $metric_source);
$analysis_source = preg_replace("/'(?:\\\\.|[^'\\\\])*'/s", "''", $analysis_source);
if ($analysis_source === null) $analysis_source = $metric_source;
$signal_count = preg_match_all(
'/\b(?:if|else|for|while|do|switch|case|return|break|continue|cin|cout|scanf|printf|input|print)\b|(?<![=!<>])=(?!=)|\+\+|--/i',
$metric_source,
'/\b(?:if|else|for|while|do|switch|case|return|break|continue|cin|cout|scanf|printf|input|print|read|readln|readline|read_line|write|writeln|writeline|readFileSync)\b|(?<![=!<>])=(?!=)|:=|\+\+|--/i',
$analysis_source,
$unused
);
if ($signal_count === false) $signal_count = 0;
$char_score = min(1, $non_space_chars / 240);
$line_score = min(1, $meaningful_lines / 10);
$activity_score = min(1, $activity_lines / 8);
$signal_score = min(1, $signal_count / 8);
$source_score = 0.25 * $char_score
+ 0.20 * $line_score
+ 0.35 * $activity_score
+ 0.20 * $signal_score;
// Completion is based on observable program stages, not source length.
// This keeps legitimate one-line Python/C++ programs out of the
// "blank submission" bucket while still catching templates and
// constant-answer fishing.
$has_input = preg_match(
'/\b(?:cin|scanf|fscanf|getchar|gets|fgets|input|read(?:ln|_?line)?|readfilesync|prompt|next(?:int|long|double|float|line)?|scanner|stdin|fmt\s*\.\s*scan\w*)\b/i',
$analysis_source
) === 1;
$has_output = preg_match(
'/\b(?:cout|cerr|printf|fprintf|puts|putchar|print|println|write(?:ln|line)?|console\s*\.\s*log|system\s*\.\s*out|fmt\s*\.\s*print\w*|echo)\b/i',
$analysis_source
) === 1;
$has_processing = preg_match(
'/\b(?:if|else|for|while|do|switch|case|sort|sum|max|min|count|reverse|reduce|filter|map)\b|'
. '(?:\+\+|--|:=|==|!=|<=|>=|&&|\|\|)|'
. '(?:[\w\]\)][ \t]*(?:\+|-|\*|\/|%)[ \t]*[\w\[\(])|'
. '(?:(?<!<)<(?![<=])|(?<!>)>(?![>=])|(?<!\|)\|(?!\|)|[&^~]|\?[^:\r\n]+:)|'
. '(?:\[[^\]\r\n]*:[^\]\r\n]*\])/i',
$analysis_source
) === 1;
$stage_count = intval($has_input) + intval($has_processing) + intval($has_output);
$constant_output_only = $has_output && !$has_input && !$has_processing;
$template_only = !$has_input
&& !$has_output
&& !$has_processing
&& ($non_space_chars < 120 || $signal_count <= 1);
$trivial = $non_space_chars < 5 || $constant_output_only || $template_only;
$trivial = $non_space_chars < 35
|| $meaningful_lines < 2
|| $activity_lines < 2;
$source_score = 0.12
+ 0.22 * intval($has_input)
+ 0.30 * intval($has_processing)
+ 0.22 * intval($has_output)
+ min(0.14, $activity_lines * 0.02);
if ($trivial) $source_score = min($source_score, 0.20);
return array(
@@ -75,6 +100,12 @@ if (!function_exists('llm_guidance_source_metrics')) {
'meaningful_lines' => $meaningful_lines,
'activity_lines' => $activity_lines,
'signal_count' => $signal_count,
'has_input' => $has_input,
'has_processing' => $has_processing,
'has_output' => $has_output,
'stage_count' => $stage_count,
'constant_output_only' => $constant_output_only,
'template_only' => $template_only,
'source_score' => max(0, min(1, $source_score)),
'trivial' => $trivial,
);
@@ -104,64 +135,49 @@ if (!function_exists('llm_guidance_assess')) {
));
}
// A nearly empty/template-only submission is always treated as low
// completion, even if other metadata is malformed or stale.
if ($result_code === 5 && $metrics['non_space_chars'] >= 5) {
// PE is judge-proven evidence that the produced content is present;
// even a compact one-line program should receive the format diff.
// CE locations come only from the compiler and the student's own
// source. They are useful and non-spoiling even for very short code.
if ($result_code === 11) {
$level = $metrics['trivial'] ? 'low' : 'medium';
$combined = $metrics['trivial'] ? 0.20 : max(0.45, $metrics['source_score']);
$mode = 'compile_location';
} elseif ($result_code === 5 && $metrics['has_output']) {
// PE is judge-proven evidence that meaningful output exists. Even
// a compact program should go straight to the format comparison.
$level = 'high';
$combined = 0.72;
} elseif ($pass_fraction >= 0.60 && $metrics['non_space_chars'] >= 5) {
// A high judge-reported pass rate is authoritative progress and
// cannot be manufactured by padding the submitted source.
$level = 'high';
$combined = max(0.70, $pass_fraction);
$mode = 'output_diff';
} elseif ($metrics['trivial']) {
$level = 'low';
$combined = min(0.20, $metrics['source_score']);
$mode = 'flowchart';
} else {
// Some judges run in non-OI mode and persist pass_rate=0 for every
// non-AC result. In that case fall back to source completeness
// instead of treating every substantive attempt as a blank page.
$combined = $pass_fraction > 0
? 0.45 * $metrics['source_score'] + 0.55 * $pass_fraction
: $metrics['source_score'];
if ($result_code === 5) {
// Presentation Error already means the computed content is
// substantially present; the useful aid is the output diff.
// The judge stops at the first failed testcase, so pass_rate=0 is
// not negative evidence. A complete input/process/output pipeline
// is enough to treat the submission as a real attempt.
$combined = max($metrics['source_score'], min(0.85, 0.45 + 0.40 * $pass_fraction));
$complete_pipeline = intval($metrics['stage_count']) === 3;
$positive_judge_evidence = $pass_fraction > 0
&& (intval($metrics['stage_count']) >= 2 || $pass_fraction >= 0.20);
if (($result_code === 5 || $result_code === 6)
&& ($complete_pipeline || $positive_judge_evidence)) {
$level = 'high';
$combined = max($combined, 0.72);
} elseif ($pass_fraction >= 0.60) {
$level = 'high';
} elseif ($pass_fraction <= 0.05 && $metrics['source_score'] < 0.45) {
$level = 'low';
} else {
$level = 'medium';
}
}
if ($level === 'low') $combined = min($combined, 0.35);
if ($level === 'medium') $combined = min($combined, 0.59);
if ($level === 'high') $combined = max($combined, 0.70);
if ($level === 'medium') $combined = min($combined, 0.69);
if ($level === 'high') $combined = max($combined, 0.70);
if ($result_code === 11
&& $level === 'low'
&& $metrics['non_space_chars'] >= 50
&& $metrics['signal_count'] >= 2) {
// Minified/one-line real programs can still have useful compiler
// locations; short garbage remains in the low flowchart path.
$level = 'medium';
$combined = max($combined, 0.40);
}
if ($result_code === 11) {
$mode = $level === 'low' ? 'flowchart' : 'compile_location';
} elseif ($level === 'low') {
$mode = 'flowchart';
} elseif ($level === 'high' && ($result_code === 5 || $result_code === 6)) {
$mode = 'output_diff';
} else {
$mode = 'focused_hint';
if ($level === 'low') {
$mode = 'flowchart';
} elseif ($level === 'high' && ($result_code === 5 || $result_code === 6)) {
$mode = 'output_diff';
} else {
$mode = 'focused_hint';
}
}
return array_merge($metrics, array(
@@ -181,6 +197,7 @@ if (!function_exists('llm_guidance_mode_label')) {
'flowchart' => '流程图引导',
'focused_hint' => '聚焦排错',
'output_diff' => '输出差异定位',
'judge_output' => '评测输出定位',
'compile_location' => '编译错误定位',
);
return isset($labels[$mode]) ? $labels[$mode] : 'AI 指导';
@@ -210,6 +227,11 @@ if (!function_exists('llm_guidance_parse_diff_line')) {
if (preg_match('/^(.*)\s+\|\s*(.*)$/', $line, $match)) {
return array(trim($match[1]), trim($match[2]));
}
// Legacy diff -y output can use a whitespace-surrounded backslash as
// the changed-line delimiter.
if (preg_match('/^(.*)\s+\\\\\s+(.*)$/', $line, $match)) {
return array(trim($match[1]), trim($match[2]));
}
if (preg_match('/^(.*?)\s+([<>])\s+(.*)$/', $line, $match)) {
return array($match[1], $match[3]);
}
@@ -245,6 +267,60 @@ if (!function_exists('llm_guidance_parse_diff_rows')) {
}
}
if (!function_exists('llm_guidance_parse_diff_blocks')) {
/**
* Parse both current simple Markdown blocks and legacy =====[file]=====
* blocks. Full diff mode is rejected before any content is interpreted.
*/
function llm_guidance_parse_diff_blocks($text) {
$text = str_replace(array("\r\n", "\r"), "\n", strval($text));
$blocks = array();
$seen = array();
if (trim($text) === '' || llm_guidance_has_full_diff_sections($text)) return $blocks;
$append_block = function($name, $body) use (&$blocks, &$seen) {
$name = trim(strval($name));
if (llm_guidance_testcase_base($name) === null) return;
$rows = llm_guidance_parse_diff_rows($body);
if (empty($rows['expected'])) return;
$key = hash('sha256', $name . "\0" . serialize($rows));
if (isset($seen[$key])) return;
$seen[$key] = true;
$blocks[] = array(
'name' => $name,
'expected' => $rows['expected'],
'yours' => $rows['yours'],
'full_mode' => false,
);
};
// A blank line before the closing ===== line is valid legacy output,
// so do not split these blocks on blank lines first.
if (preg_match_all(
'/^=+\[([^\]\r\n]+)\]=+[ \t]*\n(.*?)^[ \t]*=+[ \t]*$/ms',
$text,
$legacy_blocks,
PREG_SET_ORDER
)) {
foreach ($legacy_blocks as $legacy_block) {
$append_block($legacy_block[1], trim($legacy_block[2], "\n\r"));
}
}
// Current simple mode separates testcase blocks with a blank line.
$chunks = preg_split('/\n\s*\n/', $text);
foreach ($chunks as $chunk) {
$chunk = trim($chunk, "\r\n");
if ($chunk === '') continue;
if (preg_match('/^([^\n=|]+)\n--\n([\s\S]+)$/', $chunk, $simple_block)) {
$append_block($simple_block[1], $simple_block[2]);
}
}
return $blocks;
}
}
if (!function_exists('llm_guidance_output_attempt')) {
/**
* Inspect judge-generated Expected/Yours rows without returning either
@@ -381,6 +457,18 @@ if (!function_exists('llm_guidance_has_full_diff_sections')) {
}
}
if (!function_exists('llm_guidance_redact_full_diff')) {
/**
* Fail closed for full mode. Its separator-looking lines can also occur
* inside testcase content, so parsing the same untrusted blob cannot safely
* recover only the student's section.
*/
function llm_guidance_redact_full_diff($runtime_info) {
return '评测器生成了完整差异记录，其中包含隐藏测试输入和期望输出，'
. '已按当前账号权限省略。管理员或源码浏览权限账号可查看原文。';
}
}
if (!function_exists('llm_guidance_refine_with_output_attempt')) {
function llm_guidance_refine_with_output_attempt($assessment, $result_code, $attempt) {
$result_code = intval($result_code);
@@ -390,30 +478,15 @@ if (!function_exists('llm_guidance_refine_with_output_attempt')) {
if (empty($attempt['available'])) {
if ($assessment['mode'] === 'output_diff') {
$assessment['mode'] = 'focused_hint';
$assessment['mode'] = 'judge_output';
$assessment['show_output_diff'] = false;
}
return $assessment;
}
$has_trusted_progress = isset($assessment['judge_pass_rate'])
&& floatval($assessment['judge_pass_rate']) >= 0.60;
$empty_output = intval($attempt['yours_chars']) === 0;
$very_low_line_coverage = intval($attempt['expected_nonempty_lines']) >= 3
&& intval($attempt['yours_nonempty_lines']) * 4 < intval($attempt['expected_nonempty_lines']);
$very_low_char_coverage = intval($attempt['expected_chars']) >= 12
&& floatval($attempt['ratio']) < 0.12;
if (!$has_trusted_progress
&& ($empty_output || $very_low_line_coverage || $very_low_char_coverage)) {
$assessment['score'] = min(20, intval($assessment['score']));
$assessment['level'] = 'low';
$assessment['mode'] = 'flowchart';
$assessment['show_output_diff'] = false;
return $assessment;
}
// A single failing testcase can legitimately produce no output (for
// example, an early return or runtime fault). It is evidence about that
// testcase, not proof that the whole submission is a blank answer.
return $assessment;
}
}
@@ -425,7 +498,11 @@ if (!function_exists('llm_guidance_render_focus_payload')) {
*/
function llm_guidance_render_focus_payload($payload, $source) {
$data = json_decode(trim(strval($payload)), true);
if (!is_array($data) || !isset($data['category'])) return null;
if (!is_array($data)
|| !isset($data['category'])
|| !isset($data['confidence'])) {
return null;
}
$templates = array(
'input' => array('输入读取', '核对读取顺序、数量和数据类型，再用样例逐项对应。'),
@@ -444,14 +521,17 @@ if (!function_exists('llm_guidance_render_focus_payload')) {
$category = strval($data['category']);
if (!isset($templates[$category])) return null;
$confidence = strval($data['confidence']);
if (!in_array($confidence, array('low', 'medium', 'high'), true)) return null;
$line = isset($data['line']) ? intval($data['line']) : 0;
$source_lines = preg_split('/\r\n|\r|\n/', strval($source));
if ($line < 1 || $line > count($source_lines)) $line = 0;
if ($confidence !== 'high') $line = 0;
$where = $line > 0
? '第 ' . $line . ' 行附近：' . $templates[$category][0]
: $templates[$category][0];
return "### 定位\n" . $where . "\n\n### 检查\n" . $templates[$category][1];
return "### 建议检查\n" . $where . '：' . $templates[$category][1];
}
}