feat(llm): 引入基于完成度的 AI 提交指导与限流

- 新增 llm_guidance 策略层,根据源码、判题结果与通过率评估完成度分级
- llm-review.php 改为服务端缓冲上游输出,仅下发白名单分类的固定模板
- 新增 llm_review 与 llm_review_rate_limit 表,实现结果缓存与账户/提交两级限流
- ceinfo 模板渲染编译器标出的源码行号/列号,reinfo 仅向高完成度提交展示结构化 diff
- download.php 按完成度授权测试点下载,封堵白卷提交套取隐藏用例
- judge_client 在 ACM 非比赛场景下记录已通过测试点数,前端可显示进度
- 附 llm_guidance_policy_test 单测覆盖空白卷、注释卷、隐藏 SPJ、OJ_FULL_DIFF 等关键路径
This commit is contained in:
2026-07-23 10:47:50 +08:00
parent 5a27c07bba
commit 12bea84d8b
12 changed files with 1721 additions and 768 deletions
+577
View File
@@ -0,0 +1,577 @@
<?php
/**
* Shared, side-effect-free policy helpers for submission guidance.
*
* Keep this file independent from the database so the policy can be tested by
* the PHP CLI without loading the HUSTOJ runtime.
*/
if (!function_exists('llm_guidance_is_terminal_error')) {
function llm_guidance_is_terminal_error($result_code) {
$result_code = intval($result_code);
return $result_code >= 5 && $result_code <= 11;
}
}
if (!function_exists('llm_guidance_source_metrics')) {
function llm_guidance_source_metrics($source) {
$source = str_replace(array("\r\n", "\r"), "\n", strval($source));
$clean = preg_replace('!/\*.*?\*/!s', ' ', $source);
$clean = preg_replace('/\/\/[^\n]*/', ' ', $clean);
if ($clean === null) $clean = $source;
$non_space = preg_replace('/\s+/', '', $clean);
$non_space_chars = strlen($non_space === null ? '' : $non_space);
$lines = explode("\n", $clean);
$meaningful_lines = 0;
$activity_lines = 0;
$metric_lines = array();
foreach ($lines as $line) {
$line = trim($line);
if ($line === '') continue;
// Preprocessor directives and # comments are not student logic.
// This also prevents Python/Ruby/shell comment-only submissions
// from inflating completion metrics.
if (strpos($line, '#') === 0) continue;
if (preg_match('/^using\s+namespace\b.*;$/i', $line)) continue;
if (preg_match('/^[{};]+$/', $line)) continue;
$meaningful_lines++;
$metric_lines[] = $line;
$activity = preg_replace('/\breturn\s+(?:0|null|none)\s*;?$/i', '', $line);
$activity = preg_replace('/\b(?:int|void)\s+main\s*\([^)]*\)\s*\{?$/i', '', $activity);
$activity = trim($activity);
if ($activity !== '' && !preg_match('/^(?:public|private|protected)?\s*(?:class|struct)\s+\w+\s*\{?$/i', $activity)) {
$activity_lines++;
}
}
$metric_source = implode("\n", $metric_lines);
$signal_count = preg_match_all(
'/\b(?:if|else|for|while|do|switch|case|return|break|continue|cin|cout|scanf|printf|input|print)\b|(?<![=!<>])=(?!=)|\+\+|--/i',
$metric_source,
$unused
);
if ($signal_count === false) $signal_count = 0;
$char_score = min(1, $non_space_chars / 240);
$line_score = min(1, $meaningful_lines / 10);
$activity_score = min(1, $activity_lines / 8);
$signal_score = min(1, $signal_count / 8);
$source_score = 0.25 * $char_score
+ 0.20 * $line_score
+ 0.35 * $activity_score
+ 0.20 * $signal_score;
$trivial = $non_space_chars < 35
|| $meaningful_lines < 2
|| $activity_lines < 2;
if ($trivial) $source_score = min($source_score, 0.20);
return array(
'non_space_chars' => $non_space_chars,
'meaningful_lines' => $meaningful_lines,
'activity_lines' => $activity_lines,
'signal_count' => $signal_count,
'source_score' => max(0, min(1, $source_score)),
'trivial' => $trivial,
);
}
}
if (!function_exists('llm_guidance_assess')) {
/**
* Return the guidance mode without consulting a reference solution.
* pass_rate accepts either 0..1 (database form) or 0..100.
*/
function llm_guidance_assess($source, $result_code, $pass_rate) {
$result_code = intval($result_code);
$metrics = llm_guidance_source_metrics($source);
$pass_fraction = floatval($pass_rate);
if ($pass_fraction > 1) $pass_fraction /= 100;
$pass_fraction = max(0, min(1, $pass_fraction));
if (!llm_guidance_is_terminal_error($result_code)) {
return array_merge($metrics, array(
'score' => 0,
'level' => 'none',
'mode' => 'hidden',
'judge_pass_rate' => $pass_fraction,
'auto_fetch' => false,
'show_output_diff' => false,
));
}
// A nearly empty/template-only submission is always treated as low
// completion, even if other metadata is malformed or stale.
if ($result_code === 5 && $metrics['non_space_chars'] >= 5) {
// PE is judge-proven evidence that the produced content is present;
// even a compact one-line program should receive the format diff.
$level = 'high';
$combined = 0.72;
} elseif ($pass_fraction >= 0.60 && $metrics['non_space_chars'] >= 5) {
// A high judge-reported pass rate is authoritative progress and
// cannot be manufactured by padding the submitted source.
$level = 'high';
$combined = max(0.70, $pass_fraction);
} elseif ($metrics['trivial']) {
$level = 'low';
$combined = min(0.20, $metrics['source_score']);
} else {
// Some judges run in non-OI mode and persist pass_rate=0 for every
// non-AC result. In that case fall back to source completeness
// instead of treating every substantive attempt as a blank page.
$combined = $pass_fraction > 0
? 0.45 * $metrics['source_score'] + 0.55 * $pass_fraction
: $metrics['source_score'];
if ($result_code === 5) {
// Presentation Error already means the computed content is
// substantially present; the useful aid is the output diff.
$level = 'high';
$combined = max($combined, 0.72);
} elseif ($pass_fraction >= 0.60) {
$level = 'high';
} elseif ($pass_fraction <= 0.05 && $metrics['source_score'] < 0.45) {
$level = 'low';
} else {
$level = 'medium';
}
}
if ($level === 'low') $combined = min($combined, 0.35);
if ($level === 'medium') $combined = min($combined, 0.59);
if ($level === 'high') $combined = max($combined, 0.70);
if ($result_code === 11
&& $level === 'low'
&& $metrics['non_space_chars'] >= 50
&& $metrics['signal_count'] >= 2) {
// Minified/one-line real programs can still have useful compiler
// locations; short garbage remains in the low flowchart path.
$level = 'medium';
$combined = max($combined, 0.40);
}
if ($result_code === 11) {
$mode = $level === 'low' ? 'flowchart' : 'compile_location';
} elseif ($level === 'low') {
$mode = 'flowchart';
} elseif ($level === 'high' && ($result_code === 5 || $result_code === 6)) {
$mode = 'output_diff';
} else {
$mode = 'focused_hint';
}
return array_merge($metrics, array(
'score' => intval(round(max(0, min(1, $combined)) * 100)),
'level' => $level,
'mode' => $mode,
'judge_pass_rate' => $pass_fraction,
'auto_fetch' => true,
'show_output_diff' => $mode === 'output_diff',
));
}
}
if (!function_exists('llm_guidance_mode_label')) {
function llm_guidance_mode_label($mode) {
$labels = array(
'flowchart' => '流程图引导',
'focused_hint' => '聚焦排错',
'output_diff' => '输出差异定位',
'compile_location' => '编译错误定位',
);
return isset($labels[$mode]) ? $labels[$mode] : 'AI 指导';
}
}
if (!function_exists('llm_guidance_parse_diff_line')) {
/** Parse one judge-generated Expected/Yours row. */
function llm_guidance_parse_diff_line($line) {
$line = rtrim(strval($line), "\r\n");
if ($line === '') return null;
if (isset($line[0]) && $line[0] === '|') {
$rest = substr($line, 1);
// judge_client normalizes pipes in the student's output but not in
// the expected output, so the final pipe is the field delimiter.
$separator = strrpos($rest, '|');
if ($separator !== false) {
return array(
substr($rest, 0, $separator),
substr($rest, $separator + 1),
);
}
}
// diff -y format: greedily select the last spaced pipe delimiter.
if (preg_match('/^(.*)\s+\|\s*(.*)$/', $line, $match)) {
return array(trim($match[1]), trim($match[2]));
}
if (preg_match('/^(.*?)\s+([<>])\s+(.*)$/', $line, $match)) {
return array($match[1], $match[3]);
}
return null;
}
}
if (!function_exists('llm_guidance_parse_diff_rows')) {
/** Parse the rows after a judge Expected/Yours header. */
function llm_guidance_parse_diff_rows($body) {
$body = str_replace(array("\r\n", "\r"), "\n", strval($body));
$lines = explode("\n", $body);
$expected = array();
$yours = array();
$found_header = false;
foreach ($lines as $line) {
if (!$found_header && preg_match('/^\|?\s*Expected\s*\|\s*Yours\s*\|?\s*$/i', trim($line))) {
$found_header = true;
continue;
}
// The Markdown separator is metadata even after the header flag
// has been set; never display it as a fake first diff row.
if (preg_match('/^\s*\|?\s*-+\s*\|\s*-+\s*\|?\s*$/', $line)) continue;
if (!$found_header || $line === '') continue;
$parsed = llm_guidance_parse_diff_line($line);
if ($parsed === null) continue;
$expected[] = $parsed[0];
$yours[] = $parsed[1];
}
return array('expected' => $expected, 'yours' => $yours);
}
}
if (!function_exists('llm_guidance_output_attempt')) {
/**
* Inspect judge-generated Expected/Yours rows without returning either
* value to callers. This catches padded/template submissions that produced
* no meaningful output even when their source text is long.
*/
function llm_guidance_output_attempt($runtime_info) {
$lines = preg_split('/\r\n|\r|\n/', strval($runtime_info));
$after_header = false;
$pairs = 0;
$expected_chars = 0;
$yours_chars = 0;
$expected_nonempty_lines = 0;
$yours_nonempty_lines = 0;
foreach ($lines as $line) {
$line = rtrim($line, "\r\n");
if (preg_match('/Expected.*\|.*Yours/i', $line)) {
$after_header = true;
continue;
}
if (!$after_header || trim($line) === '') continue;
if (preg_match('/^\s*=+\s*$/', $line)) {
$after_header = false;
continue;
}
if (preg_match('/^\s*\|?\s*-+\s*\|\s*-+\s*\|?\s*$/', $line)) continue;
$parsed = llm_guidance_parse_diff_line($line);
if ($parsed === null) continue;
$expected = $parsed[0];
$yours = $parsed[1];
$expected_compact = preg_replace('/\s+/', '', $expected);
$yours_compact = preg_replace('/\s+/', '', $yours);
$expected_length = strlen($expected_compact === null ? '' : $expected_compact);
$yours_length = strlen($yours_compact === null ? '' : $yours_compact);
$pairs++;
$expected_chars += $expected_length;
$yours_chars += $yours_length;
if ($expected_length > 0) $expected_nonempty_lines++;
if ($yours_length > 0) $yours_nonempty_lines++;
}
$ratio = $expected_chars > 0
? min(1, $yours_chars / $expected_chars)
: ($yours_chars > 0 ? 1 : 0);
return array(
'available' => $pairs > 0,
'pairs' => $pairs,
'expected_chars' => $expected_chars,
'yours_chars' => $yours_chars,
'expected_nonempty_lines' => $expected_nonempty_lines,
'yours_nonempty_lines' => $yours_nonempty_lines,
'ratio' => $ratio,
);
}
}
if (!function_exists('llm_guidance_testcase_base')) {
/**
* Normalize a judge testcase filename/request to its safe basename.
* Only conservative ASCII names are accepted because the value is later
* used to open files below OJ_DATA.
*/
function llm_guidance_testcase_base($candidate) {
$candidate = trim(strval($candidate));
if (preg_match('/\.out$/i', $candidate)) {
$candidate = substr($candidate, 0, -4);
}
if ($candidate === ''
|| strpos($candidate, '..') !== false
|| !preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]*$/', $candidate)) {
return null;
}
return $candidate;
}
}
if (!function_exists('llm_guidance_diff_testcase_names')) {
/**
* Return only testcase basenames that the judge recorded for this SID.
* Expected/user output values are intentionally not returned.
*/
function llm_guidance_diff_testcase_names($runtime_info) {
$text = str_replace(array("\r\n", "\r"), "\n", strval($runtime_info));
$lines = explode("\n", $text);
$names = array();
$seen = array();
$count = count($lines);
for ($i = 0; $i < $count; $i++) {
$line = rtrim($lines[$i]);
$candidate = null;
// OJ_FULL_DIFF header. The caller separately decides whether full
// mode is safe to expose; recognizing its name is still useful for
// diagnostics and tests.
if (preg_match('/^=+\[([^\]\r\n]+\.out)\]=+\s*$/i', $line, $match)) {
$candidate = $match[1];
} elseif ($i + 2 < $count
&& trim($lines[$i + 1]) === '--'
&& preg_match('/^\|?\s*Expected\s*\|\s*Yours\s*\|?\s*$/i', trim($lines[$i + 2]))) {
// Simple diff blocks begin at the start of the runtime text or
// after the blank separator written by judge_client.
$at_boundary = $i === 0 || trim($lines[$i - 1]) === '';
if ($at_boundary && preg_match('/\.out\s*$/i', $line)) {
$candidate = trim($line);
}
}
if ($candidate === null || !preg_match('/\.out$/i', $candidate)) continue;
$base = llm_guidance_testcase_base($candidate);
if ($base === null || isset($seen[$base])) continue;
$seen[$base] = true;
$names[] = $base;
}
return $names;
}
}
if (!function_exists('llm_guidance_has_full_diff_sections')) {
/** Full mode contains hidden test input and must not reach normal users. */
function llm_guidance_has_full_diff_sections($runtime_info) {
$text = strval($runtime_info);
// runtimeinfo is capped, so later sections may be truncated. One
// distinctive full-mode marker is enough to fail closed. Accept both
// a real newline and shells that preserve the literal "\n" prefix.
return preg_match(
'/(?:^|[\r\n]|\\\\n)\s*-{3,}\s*(?:(?:test in|test out|user out) top|diff out) \d+ lines\s*-{3,}/i',
$text
) === 1;
}
}
if (!function_exists('llm_guidance_refine_with_output_attempt')) {
function llm_guidance_refine_with_output_attempt($assessment, $result_code, $attempt) {
$result_code = intval($result_code);
if ($result_code !== 5 && $result_code !== 6) {
return $assessment;
}
if (empty($attempt['available'])) {
if ($assessment['mode'] === 'output_diff') {
$assessment['mode'] = 'focused_hint';
$assessment['show_output_diff'] = false;
}
return $assessment;
}
$has_trusted_progress = isset($assessment['judge_pass_rate'])
&& floatval($assessment['judge_pass_rate']) >= 0.60;
$empty_output = intval($attempt['yours_chars']) === 0;
$very_low_line_coverage = intval($attempt['expected_nonempty_lines']) >= 3
&& intval($attempt['yours_nonempty_lines']) * 4 < intval($attempt['expected_nonempty_lines']);
$very_low_char_coverage = intval($attempt['expected_chars']) >= 12
&& floatval($attempt['ratio']) < 0.12;
if (!$has_trusted_progress
&& ($empty_output || $very_low_line_coverage || $very_low_char_coverage)) {
$assessment['score'] = min(20, intval($assessment['score']));
$assessment['level'] = 'low';
$assessment['mode'] = 'flowchart';
$assessment['show_output_diff'] = false;
return $assessment;
}
return $assessment;
}
}
if (!function_exists('llm_guidance_render_focus_payload')) {
/**
* Convert an allowlisted model classification into fixed server wording.
* No free-form model prose is ever returned to the browser.
*/
function llm_guidance_render_focus_payload($payload, $source) {
$data = json_decode(trim(strval($payload)), true);
if (!is_array($data) || !isset($data['category'])) return null;
$templates = array(
'input' => array('输入读取', '核对读取顺序、数量和数据类型,再用样例逐项对应。'),
'output' => array('输出生成', '检查是否多输出文字、少输出内容,或格式与题目要求不一致。'),
'condition' => array('条件判断', '用一组边界输入分别判断条件为真和为假时会走到哪里。'),
'loop' => array('循环控制', '只检查循环的起点、终点和每轮更新,手算前两轮与最后一轮。'),
'boundary' => array('边界处理', '分别代入最小值、最大值和临界值,观察下标与条件是否越界。'),
'initialization' => array('初值与重置', '检查变量第一次使用前的值,以及多轮处理之间是否需要重新初始化。'),
'type' => array('数据类型', '估算变量可能达到的范围,并检查运算中的类型转换。'),
'runtime' => array('运行时位置', '用最小输入复现,查看数组下标、除数和对象状态。'),
'complexity' => array('执行次数', '按最大数据范围估算最内层语句会执行多少次。'),
'memory' => array('内存使用', '核对数组或容器的规模、作用域和重复分配。'),
'state' => array('变量状态', '逐步记录关键变量,找到第一次与手算结果不同的位置。'),
'unknown' => array('最小复现', '选一组最小输入逐步手算,定位第一次偏离预期的位置。'),
);
$category = strval($data['category']);
if (!isset($templates[$category])) return null;
$line = isset($data['line']) ? intval($data['line']) : 0;
$source_lines = preg_split('/\r\n|\r|\n/', strval($source));
if ($line < 1 || $line > count($source_lines)) $line = 0;
$where = $line > 0
? '第 ' . $line . ' 行附近:' . $templates[$category][0]
: $templates[$category][0];
return "### 定位\n" . $where . "\n\n### 检查\n" . $templates[$category][1];
}
}
if (!function_exists('llm_guidance_limit_text')) {
function llm_guidance_limit_text($text, $max_length) {
$text = trim(strval($text));
$max_length = max(1, intval($max_length));
if (function_exists('mb_strlen') && function_exists('mb_substr')) {
return mb_strlen($text, 'UTF-8') > $max_length
? mb_substr($text, 0, $max_length, 'UTF-8') . '…'
: $text;
}
return strlen($text) > $max_length
? substr($text, 0, $max_length) . '...'
: $text;
}
}
if (!function_exists('llm_guidance_extract_compile_locations')) {
/**
* Parse common GCC/Clang/Javac/MSVC/Python locations and return excerpts
* from the student's own source only. No corrected/reference code exists
* in this data structure.
*/
function llm_guidance_extract_compile_locations($error_text, $source, $context_lines, $max_locations) {
$context_lines = max(0, intval($context_lines));
$max_locations = max(1, intval($max_locations));
$source = str_replace(array("\r\n", "\r"), "\n", strval($source));
$source_lines = explode("\n", $source);
$error_lines = preg_split('/\r\n|\r|\n/', strval($error_text));
$locations = array();
$seen = array();
$pending_message = '编译错误';
foreach ($error_lines as $error_line) {
$line_number = 0;
$column_number = 0;
$message = '';
if (preg_match('/^\s*(?:fatal\s+)?error(?:\[[^\]]+\])?:\s*(.+)$/i', $error_line, $headline)) {
$pending_message = $headline[1];
}
// GCC, Clang and javac: file.cpp:12:7: error: message
if (preg_match('/^.*?:(\d+):(?:(\d+):)?\s*(?:fatal\s+)?error:\s*(.+)$/i', $error_line, $match)) {
$line_number = intval($match[1]);
$column_number = isset($match[2]) ? intval($match[2]) : 0;
$message = $match[3];
}
// Free Pascal: main.pas(12,7) Error: message
elseif (preg_match('/^.*?\((\d+)(?:,(\d+))?\)\s*(?:Fatal|Error):\s*(.+)$/i', $error_line, $match)) {
$line_number = intval($match[1]);
$column_number = isset($match[2]) ? intval($match[2]) : 0;
$message = $match[3];
}
// MSVC: file.cpp(12,7): error C2143: message
elseif (preg_match('/^.*?\((\d+)(?:,(\d+))?\)\s*:\s*(?:fatal\s+)?error[^:]*:\s*(.+)$/i', $error_line, $match)) {
$line_number = intval($match[1]);
$column_number = isset($match[2]) ? intval($match[2]) : 0;
$message = $match[3];
}
// Rust: " --> src/main.rs:12:7" follows an error headline.
elseif (preg_match('/^\s*-->\s*.*?:(\d+):(\d+)\s*$/', $error_line, $match)) {
$line_number = intval($match[1]);
$column_number = intval($match[2]);
$message = $pending_message;
}
// Python-style parser location: File "main.py", line 12
elseif (preg_match('/File\s+"[^"]+",\s*line\s+(\d+)/i', $error_line, $match)) {
$line_number = intval($match[1]);
$message = '编译器在此行附近报告语法错误';
}
// PHP: "... in Main.php on line 12"
elseif (preg_match('/\bin\s+\S+\s+on\s+line\s+(\d+)\b/i', $error_line, $match)) {
$line_number = intval($match[1]);
$message = '解释器在此行附近报告语法错误';
}
// Bash: "Main.sh: line 12: syntax error"
elseif (preg_match('/:\s*line\s+(\d+):/i', $error_line, $match)) {
$line_number = intval($match[1]);
$message = '解释器在此行附近报告语法错误';
}
// Ruby and other single-column diagnostics: Main.rb:12: syntax error
elseif (preg_match('/^.*?:(\d+):\s*(?:(?:syntax|parse)\s+error.*|.+Error:.*)$/i', $error_line, $match)) {
$line_number = intval($match[1]);
$message = '解释器在此行附近报告语法错误';
}
// Go and similar compilers: main.go:12:7: undefined: name
elseif (preg_match('/^.*?:(\d+):(\d+):\s*(.+)$/', $error_line, $match)
&& !preg_match('/^\s*(?:warning|note):/i', $match[3])) {
$line_number = intval($match[1]);
$column_number = intval($match[2]);
$message = $match[3];
}
if ($line_number < 1 || $line_number > count($source_lines)) continue;
$key = $line_number . ':' . $column_number;
if (isset($seen[$key])) continue;
$seen[$key] = true;
$start = max(1, $line_number - $context_lines);
$end = min(count($source_lines), $line_number + $context_lines);
$excerpt = array();
for ($line = $start; $line <= $end; $line++) {
$excerpt[] = array(
'number' => $line,
'text' => $source_lines[$line - 1],
'is_error' => $line === $line_number,
);
}
$locations[] = array(
'line' => $line_number,
'column' => $column_number,
'message' => '编译错误',
'excerpt' => $excerpt,
);
if (count($locations) >= $max_locations) break;
}
return $locations;
}
}