klarkxy/hustoj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2024-10-10

Browse Source
This commit is contained in:
klarkxy
2024-10-10 12:56:56 +08:00
commit 7b347c848a
2648 changed files with 643965 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.gitignore vendored Normal file
View File

@@ -0,0 +1,12 @@
web/upload/*/*
web/images
web/download
web/jr
*.o
core/judged/judged
core/judged/judged.http
core/judged/judgehub
core/judge_client/judge_client
core/judge_client/judge_client.http

3
core/judge_client/getindocker.sh Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/bash
echo "This script is only for test, all modification will be lost until you add them to the Dockerfile and run docker.sh / podman.sh "
/usr/bin/docker container run --pids-limit 100 --rm --cap-add SYS_PTRACE --net=host -v /home/judge:/home/judge -it hustoj

3909
core/judge_client/judge_client.cc Normal file
View File

File diff suppressed because it is too large Load Diff

7
core/judge_client/loggedcalls.sh Executable file
View File

@@ -0,0 +1,7 @@
CALLS=""
for x in `grep CALLID\: /home/judge/log/client.log|awk -F: '{print $4}'|awk '{print $1}'|sort -nu `
do
CALLS="$CALLS,$x"
done
echo "int LANG_CV[CALL_ARRAY_SIZE]={0$CALLS,0};"

7
core/judge_client/makefile Normal file
View File

@@ -0,0 +1,7 @@
all: judge_client.cc judge_client.http
g++ -Wall -c -I/usr/local/mysql/include/mysql -I/usr/include/mysql -DOJ_USE_MYSQL judge_client.cc
g++ -Wall -o judge_client judge_client.o -L/usr/local/mysql/lib/mysql -L/usr/lib/mysql -L/usr/lib64/mysql -lmysqlclient
judge_client.http:judge_client.cc
g++ -o judge_client.http -O2 judge_client.cc
clean:
rm judge_client judge_client.http

294
core/judge_client/ncalls.h Normal file
View File

@@ -0,0 +1,294 @@
/*
* Copyright 2008 sempr <iamsempr@gmail.com>
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
static char ncalls[512] = {
1, /*0, restart_syscall*/
3, /*1, exit*/
1, /*2, fork*/
0, /*3, read*/
0, /*4, write*/
3, /*5, open*/
0, /*6, close*/
0, /*7, waitpid*/
1, /*8, creat*/
1, /*9, link*/
1, /*10, unlink*/
0, /*11, execve*/
1, /*12, chdir*/
0, /*13, time*/
1, /*14, mknod*/
1, /*15, chmod*/
0, /*16, lchown*/
1, /*17, break*/
1, /*18, oldstat*/
0, /*19, lseek*/
0, /*20, getpid*/
1, /*21, mount*/
1, /*22, umount*/
0, /*23, setuid*/
0, /*24, getuid*/
0, /*25, stime*/
1, /*26, ptrace*/
0, /*27, alarm*/
1, /*28, oldfstat*/
1, /*29, pause*/
0, /*30, utime*/
1, /*31, stty*/
1, /*32, gtty*/
0, /*33, access*/
0, /*34, nice*/
1, /*35, ftime*/
1, /*36, sync*/
1, /*37, kill*/
1, /*38, rename*/
1, /*39, mkdir*/
1, /*40, rmdir*/
1, /*41, dup*/
1, /*42, pipe*/
0, /*43, times*/
1, /*44, prof*/
0, /*45, brk*/
0, /*46, setgid*/
0, /*47, getgid*/
1, /*48, signal*/
0, /*49, geteuid*/
0, /*50, getegid*/
0, /*51, acct*/
1, /*52, umount2*/
1, /*53, lock*/
0, /*54, ioctl*/
0, /*55, fcntl*/
1, /*56, mpx*/
1, /*57, setpgid*/
1, /*58, ulimit*/
1, /*59, oldolduname*/
1, /*60, umask*/
1, /*61, chroot*/
0, /*62, ustat*/
1, /*63, dup2*/
0, /*64, getppid*/
0, /*65, getpgrp*/
1, /*66, setsid*/
0, /*67, sigaction*/
0, /*68, sgetmask*/
0, /*69, ssetmask*/
0, /*70, setreuid*/
0, /*71, setregid*/
0, /*72, sigsuspend*/
0, /*73, sigpending*/
1, /*74, sethostname*/
1, /*75, setrlimit. It is safe. An unprivilleged process can not increase limit*/
0, /*76, getrlimit*/
1, /*77, getrusage*/
1, /*78, gettimeofday*/
1, /*79, settimeofday*/
0, /*80, getgroups*/
0, /*81, setgroups*/
1, /*82, select*/
1, /*83, symlink*/
1, /*84, oldlstat*/
0, /*85, readlink*/
0, /*86, uselib*/
1, /*87, swapon*/
1, /*88, reboot*/
1, /*89, readdir*/
0, /*90, mmap*/
0, /*91, munmap*/
0, /*92, truncate*/
0, /*93, ftruncate*/
1, /*94, fchmod*/
1, /*95, fchown*/
0, /*96, getpriority*/
0, /*97, setpriority*/
1, /*98, profil*/
1, /*99, statfs*/
1, /*100, fstatfs*/
0, /*101, ioperm*/
0, /*102, socketcall*/
1, /*103, syslog*/
1, /*104, setitimer*/
0, /*105, getitimer*/
0, /*106, stat*/
0, /*107, lstat*/
0, /*108, fstat*/
0, /*109, olduname*/
0, /*110, iopl*/
0, /*111, vhangup*/
1, /*112, idle*/
0, /*113, vm86old*/
1, /*114, wait4*/
1, /*115, swapoff*/
0, /*116, sysinfo*/
1, /*117, ipc*/
0, /*118, fsync*/
0, /*119, sigreturn*/
3, /*120, clone*/
0, /*121, setdomainname*/
0, /*122, uname*/
0, /*123, modify_ldt*/
0, /*124, adjtimex*/
0, /*125, mprotect*/
0, /*126, sigprocmask*/
1, /*127, create_module*/
1, /*128, init_module*/
1, /*129, delete_module*/
0, /*130, get_kernel_syms*/
0, /*131, quotactl*/
0, /*132, getpgid*/
0, /*133, fchdir*/
0, /*134, bdflush*/
0, /*135, sysfs*/
0, /*136, personality*/
1, /*137, afs_syscall*/
0, /*138, setfsuid*/
0, /*139, setfsgid*/
0, /*140, llseek*/
1, /*141, getdents*/
1, /*142, newselect*/
0, /*143, flock*/
0, /*144, msync*/
0, /*145, readv*/
0, /*146, writev*/
0, /*147, getsid*/
0, /*148, fdatasync*/
0, /*149, sysctl*/
0, /*150, mlock*/
0, /*151, munlock*/
0, /*152, mlockall*/
0, /*153, munlockall*/
0, /*154, sched_setparam*/
0, /*155, sched_getparam*/
0, /*156, sched_setscheduler*/
0, /*157, sched_getscheduler*/
0, /*158, sched_yield*/
0, /*159, sched_get_priority_max*/
0, /*160, sched_get_priority_min*/
0, /*161, sched_rr_get_interval*/
1, /*162, nanosleep*/
0, /*163, mremap*/
0, /*164, setresuid*/
0, /*165, getresuid*/
0, /*166, vm86*/
1, /*167, query_module*/
0, /*168, poll*/
0, /*169, nfsservctl*/
1, /*170, setresgid*/
0, /*171, getresgid*/
0, /*172, prctl*/
0, /*173, rt_sigreturn*/
0, /*174, rt_sigaction*/
0, /*175, rt_sigprocmask*/
0, /*176, rt_sigpending*/
0, /*177, rt_sigtimedwait*/
0, /*178, rt_sigqueueinfo*/
0, /*179, rt_sigsuspend*/
1, /*180, pread64*/
1, /*181, pwrite64*/
1, /*182, chown*/
0, /*183, getcwd*/
0, /*184, capget*/
0, /*185, capset*/
0, /*186, sigaltstack*/
1, /*187, sendfile*/
1, /*188, getpmsg*/
1, /*189, putpmsg*/
1, /*190, vfork*/
0, /*191, ugetrlimit*/
0, /*192, mmap2*/
0, /*193, truncate64*/
0, /*194, ftruncate64*/
0, /*195, stat64*/
0, /*196, lstat64*/
0, /*197, fstat64*/
1, /*198, lchown32*/
0, /*199, getuid32*/
0, /*200, getgid32*/
0, /*201, geteuid32*/
0, /*202, getegid32*/
1, /*203, setreuid32*/
1, /*204, setregid32*/
0, /*205, getgroups32*/
1, /*206, setgroups32*/
0, /*207, fchown32*/
1, /*208, setresuid32*/
0, /*209, getresuid32*/
1, /*210, setresgid32*/
0, /*211, getresgid32*/
1, /*212, chown32*/
1, /*213, setuid32*/
1, /*214, setgid32*/
1, /*215, setfsuid32*/
1, /*216, setfsgid32*/
0, /*217, pivot_root*/
0, /*218, mincore*/
0, /*219, madvise*/
1, /*220, madvise1*/
0, /*221, getdents64*/
1, /*222, fcntl64*/
1, /*223, unused*/
0, /*224, gettid*/
0, /*225, readahead*/
1, /*226, setxattr*/
1, /*227, lsetxattr*/
1, /*228, fsetxattr*/
1, /*229, getxattr*/
1, /*230, lgetxattr*/
1, /*231, fgetxattr*/
1, /*232, listxattr*/
1, /*233, llistxattr*/
1, /*234, flistxattr*/
1, /*235, removexattr*/
1, /*236, lremovexattr*/
1, /*237, fremovexattr*/
0, /*238, tkill*/
1, /*239, sendfile64*/
0, /*240, futex*/
1, /*241, sched_setaffinity*/
1, /*242, sched_getaffinity*/
0, /*243, set_thread_area*/
0, /*244, get_thread_area*/
1, /*245, io_setup*/
1, /*246, io_destroy*/
1, /*247, io_getevents*/
1, /*248, io_submit*/
1, /*249, io_cancel*/
1, /*250, fadvise64*/
1, /*251, unused*/
3, /*252, exit_group*/
1, /*253, lookup_dcookie*/
1, /*254, epoll_create*/
1, /*255, epoll_ctl*/
1, /*256, epoll_wait*/
1, /*257, remap_file_pages*/
0, /*258, set_tid_address*/
1, /*259, timer_create*/
1, /*260, timer_settime*/
1, /*261, timer_gettime*/
1, /*262, timer_getoverrun*/
1, /*263, timer_delete*/
1, /*264, clock_settime*/
1, /*265, clock_gettime*/
1, /*266, clock_getres*/
1, /*267, clock_nanosleep*/
1, /*268, statfs64*/
1, /*269, fstatfs64*/
1, /*270, tgkill*/
1, /*271, utimes*/
1, /*272, fadvise64_64*/
1, /*273, vserver*/
};

61
core/judge_client/okcalls.h Normal file
View File

@@ -0,0 +1,61 @@
/*
*
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/syscall.h>
#define HOJ_MAX_LIMIT -1
#define CALL_ARRAY_SIZE 512
#define LANG_C 0
#define LANG_CPP 1
#define LANG_PASCAL 2
#define LANG_JAVA 3
#define LANG_RUBY 4
#define LANG_BASH 5
#define LANG_PYTHON 6
#define LANG_PHP 7
#define LANG_PERL 8
#define LANG_CSHARP 9
#define LANG_OBJC 10
#define LANG_FREEBASIC 11
#define LANG_SCHEME 12
#define LANG_CLANG 13
#define LANG_CLANGPP 14
#define LANG_LUA 15
#define LANG_JS 16
#define LANG_GO 17
#define LANG_SQL 18
#define LANG_FORTRAN 19
#define LANG_MATLAB 20
#define LANG_COBOL 21
#define LANG_R 22
#define LANG_SB3 23
#ifdef __i386
#include "okcalls32.h"
#endif
#ifdef __x86_64
#include "okcalls64.h"
#endif
#ifdef __arm__
#include "okcalls_arm.h"
#endif
#ifdef __aarch64__
#include "okcalls_aarch64.h"
#endif
#ifdef __mips__
#include "okcalls_mips.h"
#endif

134
core/judge_client/okcalls32.h Normal file
View File

@@ -0,0 +1,134 @@
/*
*
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
//c & c++
int LANG_CV[CALL_ARRAY_SIZE] = {3,4,8,10,11,33,45,85,122,125,140,146,191,192,197,240,243,252,258,272,311,355,383,386,SYS_time, SYS_read, SYS_uname, SYS_write, SYS_open,
SYS_close, SYS_access, SYS_brk, SYS_munmap, SYS_mprotect,
SYS_mmap2, SYS_fstat64, SYS_set_thread_area, 463, 0 };
//pascal
int LANG_PV[CALL_ARRAY_SIZE] = { 0,9, 59, 97, 13, 16, 89, 140, 91, 175, 195, 13, SYS_open, SYS_set_thread_area,
SYS_brk, SYS_read, SYS_uname, SYS_write, SYS_execve, SYS_ioctl,
SYS_readlink, SYS_mmap, SYS_rt_sigaction, SYS_getrlimit, 252, 191, 0 };
//java
int LANG_JV[CALL_ARRAY_SIZE] = { 3,6,11,20,33,45,85,91,120,122,125,172,174,175,191,192,195,197,240,243,252,258,295,311
, SYS_fcntl64, SYS_getdents64, SYS_ugetrlimit,
SYS_rt_sigprocmask, SYS_futex, SYS_read, SYS_mmap2, SYS_stat64,
SYS_open, SYS_close, SYS_execve, SYS_access, SYS_brk, SYS_readlink,
SYS_munmap, SYS_close, SYS_uname, SYS_clone, SYS_uname, SYS_mprotect,
SYS_rt_sigaction, SYS_sigprocmask, SYS_getrlimit, SYS_fstat64,
SYS_getuid32, SYS_getgid32, SYS_geteuid32, SYS_getegid32,
SYS_set_thread_area, SYS_set_tid_address, SYS_set_robust_list,
SYS_exit_group, 0 };
//ruby
int LANG_RV[CALL_ARRAY_SIZE] = { 3,4,5,6,11,33,42,45,54,77,78,91,120,122,125,174,175,183,186,191,192,
195,196,197,199,200,201,202,221,240,243,252,258,311,
340, 126, SYS_access, SYS_brk, SYS_close, SYS_execve,
SYS_exit_group, SYS_fstat64, SYS_futex, SYS_getegid32, SYS_geteuid32,
SYS_getgid32, SYS_getrlimit, SYS_gettimeofday, SYS_getuid32, SYS_mmap2,
SYS_mprotect, SYS_munmap, SYS_open, SYS_read, SYS_rt_sigaction,
SYS_rt_sigprocmask, SYS_set_robust_list, SYS_set_thread_area,
SYS_set_tid_address, SYS_uname, SYS_write, 191, 195, 120, 7, 0 };
//bash
int LANG_BV[CALL_ARRAY_SIZE] = { 3,4,5,6,11,20,33,45,54,63,64,65,78,116,122,125,140,174,175,183,191,192,195,197,199,200,201,202,221,243,252,
7, 120, 42, 158, 117, 60, 39, 102, 191, 183, SYS_access,
SYS_brk, SYS_close, SYS_dup2, SYS_execve, SYS_exit_group, SYS_fcntl64,
SYS_fstat64, SYS_getegid32, SYS_geteuid32, SYS_getgid32, SYS_getpgrp,
SYS_getpid, SYS_getppid, SYS_getrlimit, SYS_gettimeofday, SYS_getuid32,
SYS_ioctl, SYS__llseek, SYS_mmap2, SYS_mprotect, SYS_munmap, SYS_open,
SYS_read, SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_set_thread_area,
SYS_stat64, SYS_time, SYS_uname, SYS_write,0 };
//python
int LANG_YV[CALL_ARRAY_SIZE]={3,4,5,6,11,20,22,33,41,45,54,85,91,116,122,125,140,174,175,183,
191,192,195,196,197,199,200,201,202,220,221,243,252,258,265,295,
311,13,41,91,102,186,221,240,295,355,0};
//php
int LANG_PHV[CALL_ARRAY_SIZE] = {3,4,5,6,11,13,33,45,54,78,91,122,125,140,174,175,183,191,192,195,
196,197,240,243,252,258,295,311,146, 158, 117, 60, 39, 102, SYS_access, SYS_brk,
SYS_clone, SYS_close, SYS_execve, SYS_exit_group, SYS_fcntl64,
SYS_fstat64, SYS_futex, SYS_getcwd, SYS_getdents64, SYS_getrlimit,
SYS_gettimeofday, SYS_ioctl, SYS__llseek, SYS_lstat64, SYS_mmap2,
SYS_mprotect, SYS_munmap, SYS_open, SYS_read, SYS_readlink,
SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_set_robust_list,
SYS_set_thread_area, SYS_set_tid_address, SYS_stat64, SYS_time,
SYS_uname, SYS_write, 0 };
//perl
int LANG_PLV[CALL_ARRAY_SIZE] = { 78, 158, 117, 60, 39, 102, 191, SYS_access, SYS_brk,
SYS_close, SYS_execve, SYS_exit_group, SYS_fcntl64, SYS_fstat64,
SYS_futex, SYS_getegid32, SYS_geteuid32, SYS_getgid32, SYS_getrlimit,
SYS_getuid32, SYS_ioctl, SYS__llseek, SYS_mmap2, SYS_mprotect,
SYS_munmap, SYS_open, SYS_read, SYS_readlink, SYS_rt_sigaction,
SYS_rt_sigprocmask, SYS_set_robust_list, SYS_set_thread_area,
SYS_set_tid_address, SYS_stat64, SYS_time, SYS_uname, SYS_write, 0 };
//c-sharp
int LANG_CSV[CALL_ARRAY_SIZE] = {3,4,5,6,11,13,33,45,54,78,85,91,99,102,120,122,125,140,141,158,174,175,
183,186,191,192,195,197,199,221,240,242,243,252,258,265,266,270,295,
311, 11,33,45,192,141, 158, 117, 60, 39, 102, 191, SYS_access, SYS_brk,
SYS_chmod, SYS_clock_getres, SYS_clock_gettime, SYS_clone, SYS_close,
SYS_execve, SYS_exit_group, SYS_fcntl64, SYS_fstat64, SYS_ftruncate64,
SYS_futex, SYS_getcwd, SYS_getdents64, SYS_geteuid32, SYS_getpid,
SYS_getppid, SYS_getrlimit, SYS_gettimeofday, SYS_getuid32, SYS_ioctl,
SYS__llseek, SYS_lstat64, SYS_mmap2, SYS_mprotect, SYS_mremap,
SYS_munmap, SYS_open, SYS_read, SYS_readlink, SYS_rt_sigaction,
SYS_rt_sigprocmask, SYS_sched_getaffinity, SYS_sched_getparam,
SYS_sched_get_priority_max, SYS_sched_get_priority_min,
SYS_sched_getscheduler, SYS_set_robust_list, SYS_set_thread_area,
SYS_set_tid_address, SYS_sigaltstack, SYS_stat64, SYS_statfs,
SYS_tgkill, SYS_time, SYS_uname, SYS_unlink, SYS_write, 0 };
//objective-c
int LANG_OV[CALL_ARRAY_SIZE] = {3,221,102, 191, SYS_access, SYS_brk, SYS_close, SYS_execve,
SYS_exit_group, SYS_fstat64, SYS_futex, SYS_getcwd, SYS_geteuid32,
SYS_getrlimit, SYS_gettimeofday, SYS_getuid32, SYS__llseek, SYS_lstat64,
SYS_mmap2, SYS_mprotect, SYS_munmap, SYS_open, SYS_read, SYS_readlink,
SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_set_robust_list,
SYS_set_thread_area, SYS_set_tid_address, SYS_stat64, SYS_uname,
SYS_write, 0 };
//freebasic
int LANG_BASICV[CALL_ARRAY_SIZE] = {3,4,5,6,11,33,45,54,91,101,122,125,140,174,175,191,192,195,197,240,243,252,258,311,330
, SYS_access, SYS_brk, SYS_close, SYS_execve,
SYS_exit_group, SYS_fstat64, SYS_futex, SYS_getrlimit, SYS_ioctl,
SYS_ioperm, SYS_mmap2, SYS_open, SYS_read, SYS_rt_sigaction,
SYS_rt_sigprocmask, SYS_set_robust_list, SYS_set_thread_area,
SYS_set_tid_address, SYS_stat64, SYS_uname, SYS_write, 0 };
//scheme
int LANG_SV[CALL_ARRAY_SIZE] = { 100, 295, SYS_fcntl64, SYS_getdents64, SYS_ugetrlimit,
SYS_rt_sigprocmask, SYS_futex, SYS_read, SYS_mmap2, SYS_stat64,
SYS_open, SYS_close, SYS_execve, SYS_access, SYS_brk, SYS_readlink,
SYS_munmap, SYS_close, SYS_uname, SYS_clone, SYS_uname, SYS_mprotect,
SYS_rt_sigaction, SYS_sigprocmask, SYS_getrlimit, SYS_fstat64,
SYS_getuid32, SYS_getgid32, SYS_geteuid32, SYS_getegid32,
SYS_set_thread_area, SYS_set_tid_address, SYS_set_robust_list,
SYS_exit_group, 0 };
//lua
int LANG_LUAV[CALL_ARRAY_SIZE]={3,4,5,6,11,13,33,45,91,125,174,192,195,197,243,252,330,0};
//nodejs
int LANG_JSV[CALL_ARRAY_SIZE]={3,4,5,6,11,33,45,54,75,78,85,91,120,122,125,168,174,175,183,191,192,195,196,197,199,200,201,202,224,240,243,252,255,256,258,265,266,311,328,329,331,0};
//go-lang
int LANG_GOV[CALL_ARRAY_SIZE]={3,4,11,120,123,174,175,186,192,240,242,252,265,0};
//sqlite3
int LANG_SQLV[CALL_ARRAY_SIZE]={3,4,5,6,11,33,45,54,102,122,125,140,174,175,183,191,192,195,197,199,221,240,243,252,258,311,0};
//fortran
int LANG_FV[CALL_ARRAY_SIZE]={3,4,11,33,45,85,122,174,197,243,252,0};
//matlab
int LANG_MV[CALL_ARRAY_SIZE]={3,4,6,11,20,33,45,54,78,85,91,102,122,125,140,146,168,174,175,183,191,192,195,196,197,199,220,221,240,242,243,252,258,295,311,355,0};
//Cobol
int LANG_CBV[CALL_ARRAY_SIZE]={3,4,6,11,20,33,45,54,78,85,91,102,122,125,140,146,168,174,175,183,191,192,195,196,197,199,220,221,240,242,243,252,258,295,311,355,0};
//R
int LANG_RLV[CALL_ARRAY_SIZE]={0,11,0};
int LANG_SB3V[CALL_ARRAY_SIZE]={0,11,0};

158
core/judge_client/okcalls64.h Normal file
View File

@@ -0,0 +1,158 @@
/*
*
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
//c & c++
int LANG_CV[CALL_ARRAY_SIZE] = {0,1,2,3,5,8,9,10,11,12,13,14,16,20,21,39,56,59,63,89,99,158,186,218,231,234,257,262,267,268,275,292,302,318,334,511,
SYS_write, SYS_mprotect, SYS_munmap, SYS_brk, SYS_arch_prctl, SYS_pread64, SYS_open, SYS_writev,
SYS_time, SYS_futex, SYS_set_thread_area, SYS_access, SYS_clock_gettime, SYS_exit_group, SYS_mq_open,
SYS_ioprio_get, SYS_unshare, SYS_set_robust_list, SYS_splice, SYS_close, SYS_stat, SYS_fstat, SYS_execve,
SYS_uname, SYS_lseek, SYS_readlink, SYS_mmap, SYS_sysinfo, 0 };
//pascal
int LANG_PV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_munmap, SYS_brk, SYS_rt_sigaction, SYS_arch_prctl, SYS_ioctl,
SYS_pread64, SYS_getxattr, SYS_open, SYS_time, SYS_set_thread_area, SYS_exit_group, SYS_ioprio_get, SYS_close,
SYS_stat, SYS_execve, SYS_uname, SYS_readlink, SYS_mmap, SYS_getrlimit, 0 };
//java
int LANG_JV[CALL_ARRAY_SIZE] = {
0, 262,318,334,435, SYS_mprotect, SYS_getuid, SYS_getgid, SYS_geteuid, SYS_getegid, SYS_munmap, SYS_getppid, SYS_getpgrp,
SYS_brk, SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_prctl, SYS_arch_prctl, SYS_ioctl, SYS_pread64, SYS_open,
SYS_futex, SYS_set_thread_area, SYS_access, SYS_getdents64, SYS_set_tid_address, SYS_pipe, SYS_exit_group,
SYS_openat, SYS_set_robust_list, SYS_close, SYS_prlimit64, SYS_dup2, SYS_getpid, SYS_stat, SYS_fstat, SYS_clone,
SYS_execve, SYS_lstat, SYS_wait4, SYS_uname, SYS_fcntl, SYS_getcwd, SYS_lseek, SYS_readlink, SYS_mmap,
SYS_getrlimit, 0 };
//ruby
int LANG_RV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_getuid, SYS_getgid, SYS_geteuid, SYS_getegid, SYS_munmap, SYS_brk,
SYS_capset, SYS_rt_sigaction, SYS_sigaltstack, SYS_rt_sigprocmask, SYS_arch_prctl, SYS_ioctl, SYS_pread64,
SYS_open, SYS_futex, SYS_access, SYS_set_tid_address, SYS_pipe, SYS_exit_group, SYS_set_robust_list, SYS_close,
SYS_stat, SYS_fstat, SYS_clone, SYS_execve, SYS_fcntl, SYS_mmap, SYS_gettimeofday, SYS_getrlimit,
SYS_getrusage, 0 };
//bash
int LANG_BV[CALL_ARRAY_SIZE] = {
0, 202,218,262,273,318,334,SYS_write, SYS_mprotect, SYS_getuid, SYS_getgid, SYS_geteuid, SYS_getegid, SYS_munmap, SYS_getppid,
SYS_getpgrp, SYS_brk, SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_arch_prctl, SYS_ioctl, SYS_pread64,
SYS_afs_syscall, SYS_getxattr, SYS_open, SYS_access, SYS_pipe, SYS_clock_nanosleep, SYS_exit_group, SYS_openat,
SYS_close, SYS_prlimit64, SYS_dup2, SYS_getpid, SYS_stat, SYS_socket, SYS_connect, SYS_fstat, SYS_clone,
SYS_execve, SYS_exit, SYS_wait4, SYS_uname, SYS_fcntl, SYS_getcwd, SYS_lseek, SYS_mmap, SYS_gettimeofday,
SYS_getrlimit, SYS_sysinfo, 0 };
//python
int LANG_YV[CALL_ARRAY_SIZE] = {
0,1,2,3,4,5,6,8,9,10,11,12,13,14,16,17,21,32,39,41,42,49,59,72,78,79,89,97,99,102,104,106,107,108,131,137,158,186,202,217,218,228,231,257,262,273,302,318,334,
511,
SYS_write, SYS_mprotect, SYS_getuid, SYS_getgid, SYS_geteuid, SYS_getegid, SYS_munmap, SYS_brk,
SYS_rt_sigaction, SYS_sigaltstack, SYS_rt_sigprocmask, SYS_sched_get_priority_max, SYS_arch_prctl, SYS_ioctl,
SYS_pread64, SYS_getxattr, SYS_open, SYS_futex, SYS_access, SYS_getdents64, SYS_set_tid_address, SYS_clock_gettime,
SYS_exit_group, SYS_mremap, SYS_openat, SYS_unshare, SYS_set_robust_list, SYS_close, SYS_prlimit64,
SYS_dup, SYS_getpid, SYS_stat, SYS_socket, SYS_connect, SYS_fstat, SYS_execve, SYS_lstat,
SYS_exit, SYS_fcntl, SYS_getdents, SYS_getcwd, SYS_lseek, SYS_readlink, SYS_mmap, SYS_getrlimit,
SYS_sysinfo, 0 };
//php
int LANG_PHV[CALL_ARRAY_SIZE] = {
0,1,3,4,5,6,8,9,10,11,12,13,14,16,17,21,59,79,99,158,202,218,231,257,262,273,302,318,334,
SYS_write, SYS_mprotect, SYS_munmap, SYS_brk, SYS_rt_sigaction, SYS_rt_sigprocmask,
SYS_sched_get_priority_max, SYS_arch_prctl, SYS_ioctl, SYS_pread64, SYS_getxattr, SYS_open, SYS_writev,
SYS_time, SYS_futex, SYS_set_thread_area, SYS_access, SYS_getdents64, SYS_set_tid_address, SYS_exit_group,
SYS_openat, SYS_set_robust_list, SYS_close, SYS_getpid, SYS_stat, SYS_fstat, SYS_clone, SYS_execve, SYS_lstat,
SYS_exit, SYS_uname, SYS_fcntl, SYS_getdents, SYS_getcwd, SYS_lseek, SYS_readlink, SYS_mmap, SYS_gettimeofday,
SYS_getrlimit, 0 };
//perl
int LANG_PLV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_getuid, SYS_getgid, SYS_geteuid, SYS_getegid, SYS_munmap, SYS_brk,
SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_arch_prctl, SYS_ioctl, SYS_pread64, SYS_getxattr, SYS_open, SYS_time,
SYS_futex, SYS_set_thread_area, SYS_access, SYS_set_tid_address, SYS_exit_group, SYS_set_robust_list, SYS_close,
SYS_getpid, SYS_stat, SYS_fstat, SYS_execve, SYS_exit, SYS_uname, SYS_fcntl, SYS_getdents, SYS_lseek,
SYS_readlink, SYS_mmap, SYS_gettimeofday, SYS_getrlimit, 0 };
//c-sharp
int LANG_CSV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_getuid, SYS_geteuid, SYS_munmap, SYS_getppid, SYS_brk, SYS_rt_sigaction,
SYS_sigaltstack, SYS_statfs, SYS_rt_sigprocmask, SYS_setpriority, SYS_sched_getparam, SYS_sched_getscheduler,
SYS_sched_get_priority_max, SYS_sched_get_priority_min, SYS_prctl, SYS_arch_prctl, SYS_ioctl, SYS_pread64,
SYS_getxattr, SYS_open, SYS_time, SYS_futex, SYS_sched_getaffinity, SYS_set_thread_area, SYS_access,
SYS_getdents64, SYS_set_tid_address, SYS_clock_gettime, SYS_clock_getres, SYS_exit_group, SYS_tgkill,
SYS_sched_yield, SYS_mremap, SYS_openat, SYS_set_robust_list, SYS_close, SYS_prlimit64, SYS_getpid, SYS_stat,
SYS_socket, SYS_connect, SYS_fstat, SYS_clone, SYS_execve, SYS_lstat, SYS_exit, SYS_uname, SYS_semget,
SYS_semop, SYS_semctl, SYS_fcntl, SYS_ftruncate, SYS_getdents, SYS_getcwd, SYS_lseek, SYS_mkdir, SYS_unlink,
SYS_readlink, SYS_mmap, SYS_chmod, SYS_umask, SYS_gettimeofday, SYS_getrlimit, 0 };
//objective-c
int LANG_OV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_getuid, SYS_munmap, SYS_brk, SYS_rt_sigaction, SYS_rt_sigprocmask,
SYS_arch_prctl, SYS_pread64, SYS_getxattr, SYS_open, SYS_futex, SYS_set_thread_area, SYS_access,
SYS_set_tid_address, SYS_exit_group, SYS_set_robust_list, SYS_close, SYS_stat, SYS_fstat, SYS_execve, SYS_uname,
SYS_getcwd, SYS_readlink, SYS_mmap, SYS_gettimeofday, SYS_getrlimit, 0 };
//freebasic
int LANG_BASICV[CALL_ARRAY_SIZE] = {
0,1,3,4,5,9,10,12,13,14,16,17,21,59,158,173,218,231,257,273,302, SYS_write, SYS_mprotect, SYS_ptrace, SYS_brk, SYS_setfsuid, SYS_capget, SYS_rt_sigaction,
SYS_rt_sigprocmask, SYS_sched_get_priority_max, SYS_arch_prctl, SYS_ioctl, SYS_pread64, SYS_ioperm,
SYS_create_module, SYS_init_module, SYS_getxattr, SYS_lgetxattr, SYS_llistxattr, SYS_removexattr, SYS_open,
SYS_futex, SYS_set_thread_area, SYS_access, SYS_set_tid_address, SYS_exit_group, SYS_mq_open,
SYS_mq_timedreceive, SYS_ioprio_get, SYS_mkdirat, SYS_set_robust_list, SYS_close, SYS_process_vm_writev,
SYS_dup2, SYS_stat, SYS_recvfrom, SYS_fstat, SYS_setsockopt, SYS_execve, SYS_lstat, SYS_uname, SYS_mmap,
SYS_getrlimit, 0 };
//scheme
int LANG_SV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_times, SYS_getuid, SYS_getgid, SYS_geteuid, SYS_getegid, SYS_munmap,
SYS_getppid, SYS_getpgrp, SYS_brk, SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_arch_prctl, SYS_ioctl, SYS_open,
SYS_futex, SYS_set_thread_area, SYS_access, SYS_getdents64, SYS_set_tid_address, SYS_pipe, SYS_select,
SYS_exit_group, SYS_set_robust_list, SYS_close, SYS_dup2, SYS_getpid, SYS_stat, SYS_fstat, SYS_clone,
SYS_execve, SYS_lstat, SYS_wait4, SYS_uname, SYS_fcntl, SYS_getcwd, SYS_lseek, SYS_readlink, SYS_mmap,
SYS_getrlimit, 0 };
//lua
int LANG_LUAV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_munmap, SYS_brk, SYS_rt_sigaction, SYS_arch_prctl, SYS_pread64, SYS_open,
SYS_access, SYS_exit_group, SYS_dup3, SYS_close, SYS_stat, SYS_fstat, SYS_execve, SYS_mmap, 0 };
//nodejs javascript
int LANG_JSV[CALL_ARRAY_SIZE] = {
0,1,3,4,5,9,10,11,12,13,14,16,17,21,28,39,56,59,79,89,102,104,107,108,158,202,218,229,231,232,233,257,273,290,291,293,302,318,332
, SYS_write, SYS_mprotect, SYS_getuid, SYS_getgid, SYS_geteuid, SYS_getegid, SYS_munmap, SYS_brk,
SYS_rt_sigaction, SYS_rt_sigprocmask, SYS_arch_prctl, SYS_ioctl, SYS_setrlimit, SYS_pread64, SYS_gettid,
SYS_open, SYS_writev, SYS_futex, SYS_access, SYS_set_tid_address, SYS_clock_gettime, SYS_clock_getres,
SYS_exit_group, SYS_epoll_wait, SYS_epoll_ctl, SYS_set_robust_list, SYS_eventfd2, SYS_epoll_create1, SYS_pipe2,
SYS_close, SYS_stat, SYS_fstat, SYS_clone, SYS_execve, SYS_lstat, SYS_poll, SYS_getcwd, SYS_readlink, SYS_mmap,
SYS_gettimeofday, SYS_getrlimit, 0 };
//go-lang
int LANG_GOV[CALL_ARRAY_SIZE] = {
0, 3, 35,511, SYS_write, SYS_mprotect, SYS_munmap, SYS_rt_sigaction, SYS_sigaltstack, SYS_rt_sigprocmask,
SYS_arch_prctl, SYS_pread64, SYS_gettid, SYS_futex, SYS_sched_getaffinity, SYS_clock_gettime, SYS_exit_group,
SYS_sched_yield, SYS_openat, SYS_readlinkat, SYS_clone, SYS_execve, SYS_fcntl, SYS_mmap, 0 };
//sqlite3
int LANG_SQLV[CALL_ARRAY_SIZE] = {0,8,11,18,75,87,1,3,4,5,6,9,10,12,13,14,16,17,21,39,41,42,59,72,79,102,107,158,218,231,257,262,273,302,318,334,
SYS_write, SYS_mprotect, SYS_getuid, SYS_geteuid, SYS_brk, SYS_rt_sigaction, SYS_rt_sigprocmask,
SYS_arch_prctl, SYS_ioctl, SYS_pread64, SYS_open, SYS_futex, SYS_access, SYS_set_tid_address, SYS_exit_group,
SYS_set_robust_list, SYS_close, SYS_stat, SYS_socket, SYS_connect, SYS_fstat, SYS_execve, SYS_semget, SYS_fcntl,
SYS_fsync, SYS_getcwd, SYS_lseek, SYS_unlink, SYS_mmap, SYS_getrlimit, 0 };
//fortran
int LANG_FV[CALL_ARRAY_SIZE] = {
0, SYS_write, SYS_mprotect, SYS_brk, SYS_rt_sigaction, SYS_arch_prctl, SYS_pread64, SYS_access,
SYS_exit_group, SYS_fstat, SYS_execve, SYS_uname, SYS_readlink, 0 };
//octave
int LANG_MV[CALL_ARRAY_SIZE] = {
0,1,3,4,5,6,8,9,10,11,12,13,14,16,17,21,39,41,42,44,47,49,51,54,56,59,63,79,89,102,158,202,204,217,218,231,257,273,302,318,
SYS_write, SYS_mprotect, SYS_getuid, SYS_munmap, SYS_brk, SYS_rt_sigaction, SYS_rt_sigprocmask,
SYS_arch_prctl, SYS_ioctl, SYS_writev, SYS_futex, SYS_sched_getaffinity, SYS_access, SYS_set_tid_address,
SYS_exit_group, SYS_openat, SYS_set_robust_list, SYS_close, SYS_prlimit64, 318, SYS_getpid, SYS_stat,
SYS_socket, SYS_connect, SYS_recvfrom, SYS_shutdown, SYS_fstat, SYS_getpeername, SYS_clone, SYS_execve,
SYS_lstat, SYS_uname, SYS_poll, SYS_fcntl, SYS_getdents, SYS_getcwd, SYS_lseek, SYS_readlink, SYS_mmap,
SYS_gettimeofday, 0 };
//Cobal
int LANG_CBV[CALL_ARRAY_SIZE]={0,1,3,4,5,8,9,10,11,12,13,14,17,21,41,42,59,89,158,202,218,231,257,273,302,0};
//R Lang
int LANG_RLV[CALL_ARRAY_SIZE]={0,1,3,8,9,10,11,12,13,14,16,17,21,33,39,41,42,56,59,61,63,72,79,83,89,98,99,102,104,107,
108,110,111,131,158,202,204,217,218,231,257,262,273,293,302,318,334,435,439,0};
int LANG_SB3V[CALL_ARRAY_SIZE]={0,1,3,5,9,10,11,12,13,14,16,17,21,28,39,56,59,63,72,89,99,102,104,107,108,158,202,218,231,232,233,257,262,273,290,291,293,302,318,332,334,435,0};

62
core/judge_client/okcalls_aarch64.h Normal file
View File

@@ -0,0 +1,62 @@
/*
*
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/syscall.h>
#define HOJ_MAX_LIMIT -1
//C C++
int LANG_CV[CALL_ARRAY_SIZE] = {0,1,27,48,56,62,63,64,72,78,80,94,104,112,134,135,160,184,200,214,216,220,221,248,152,301,344,136,120,8,40,232,328,440,424,472,360,24,456,504,296,168,376,382,392,511,280,488,88,312,264,408,0};
//pascal
int LANG_PV[CALL_ARRAY_SIZE] = {29,63,64,78,94,112,134,163,220,511,0};
//java
int LANG_JV[CALL_ARRAY_SIZE] = {48,56,57,63,78,79,80,94,96,98,99,112,134,135,163,172,179,214,215,220,222,226,511,0};
//ruby
int LANG_RV[CALL_ARRAY_SIZE] = {0,0};
//bash
int LANG_BV[CALL_ARRAY_SIZE] = {0,3,4,5,6,19,20,33,45,54,63,64,65,78,122,125,140,174,175,183,191,192,195,197,199,200,201,202,221,248,0};
//python
int LANG_YV[CALL_ARRAY_SIZE] = {0,3,4,5,6,16,19,32,33,41,45,48,54,64,80,85,91,96,112,122,125,128,140,144,160,174,175,176,183,186,191,192,195,196,197,199,200,201,202,208,217,220,221,288,224,240,248,256,272,304,320,322,336,352,368,338,432,400,480,384,416,448,496,0};
//php
int LANG_PHV[CALL_ARRAY_SIZE] = {0,0};
//perl
int LANG_PLV[CALL_ARRAY_SIZE] = {0,0};
//c-sharp
int LANG_CSV[CALL_ARRAY_SIZE] = {0,3,5,6,19,33,45,122,125,174,175,191,192,195,197,256,338,0};
//objective-c
int LANG_OV[CALL_ARRAY_SIZE] = {0,0};
//freebasic
int LANG_BASICV[CALL_ARRAY_SIZE] = {0,0};
//scheme
int LANG_SV[CALL_ARRAY_SIZE] = {0,0};
//lua
int LANG_LUAV[CALL_ARRAY_SIZE] = {0,0};
//nodejs
int LANG_JSV[CALL_ARRAY_SIZE] = {0,0};
//go-lang
int LANG_GOV[CALL_ARRAY_SIZE] = {0,0};
//sqlite3
int LANG_SQLV[CALL_ARRAY_SIZE] = {0,0};
//fortran
int LANG_FV[CALL_ARRAY_SIZE] = {0,3,4,5,33,45,85,122,174,197,248,0};
//matlib
int LANG_MV[CALL_ARRAY_SIZE] = {0,2,3,4,5,6,11,19,33,45,54,78,91,122,125,140,174,175,183,191,192,195,197,199,201,217,240,242,248,256,281,283,338,384,0};
//cobol
int LANG_CBV[CALL_ARRAY_SIZE] = {0,2,3,4,5,6,11,19,33,45,54,78,91,122,125,140,174,175,183,191,192,195,197,199,201,217,240,242,248,256,281,283,338,384,0};
//R
int LANG_RLV[CALL_ARRAY_SIZE]={0,11,0};
//Scratch3
int LANG_SB3V[CALL_ARRAY_SIZE]={0,11,0};

63
core/judge_client/okcalls_arm.h Normal file
View File

@@ -0,0 +1,63 @@
/*
*
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/syscall.h>
#define HOJ_MAX_LIMIT -1
#define LANGV_LENGTH 256
//C C++
int LANG_CV[CALL_ARRAY_SIZE]={0,3,4,5,33,45,85,122,192,197,248,0};
//pascal
int LANG_PV[CALL_ARRAY_SIZE] = {0,3,4,54,85,174,191,248,0};
//java
int LANG_JV[CALL_ARRAY_SIZE] ={0,3,5,6,11,20,33,45,85,91,120,122,125,140,174,175,191,192,195,197,240,248,256,322,338,0};
//ruby
int LANG_RV[CALL_ARRAY_SIZE] = {0,0};
//bash
int LANG_BV[CALL_ARRAY_SIZE]={0,3,4,5,6,19,20,33,45,54,63,64,65,78,122,125,140,174,175,183,191,192,195,197,199,200,201,202,221,248,0};
//python
int LANG_YV[CALL_ARRAY_SIZE] = {0,3,4,5,6,19,33,41,45,54,85,91,122,125,140,174,175,183,186,191,192,195,196,197,199,200,201,202,217,221,240,248,256,322,338,0};
//php
int LANG_PHV[CALL_ARRAY_SIZE] = {0,0};
//perl
int LANG_PLV[CALL_ARRAY_SIZE] = {0,0};
//c-sharp
int LANG_CSV[CALL_ARRAY_SIZE]={0,3,5,6,19,33,45,122,125,174,175,191,192,195,197,256,338,0};
//objective-c
int LANG_OV[CALL_ARRAY_SIZE] = {0,0};
//freebasic
int LANG_BASICV[CALL_ARRAY_SIZE] = {0,0};
//scheme
int LANG_SV[CALL_ARRAY_SIZE] = {0,0};
//lua
int LANG_LUAV[CALL_ARRAY_SIZE]={0,0};
//nodejs
int LANG_JSV[CALL_ARRAY_SIZE]={0,0};
//go-lang
int LANG_GOV[CALL_ARRAY_SIZE]={0,0};
//sqlite3
int LANG_SQLV[CALL_ARRAY_SIZE]={0,0};
//fortran
int LANG_FV[CALL_ARRAY_SIZE]={0,3,4,5,33,45,85,122,174,197,248,0};
//R
int LANG_RLV[CALL_ARRAY_SIZE]={0,11,0};
//matlib
int LANG_MV[CALL_ARRAY_SIZE]={0,2,3,4,5,6,11,19,33,45,54,78,91,122,125,140,174,175,183,191,192,195,197,199,201,217,240,242,248,256,281,283,338,384,0};
//Cobol
int LANG_CBV[CALL_ARRAY_SIZE]={0,3,4,5,33,45,85,122,192,197,248,0};
//Scratch3
int LANG_SB3V[CALL_ARRAY_SIZE]={0,11,0};

63
core/judge_client/okcalls_mips.h Normal file
View File

@@ -0,0 +1,63 @@
/*
*
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/syscall.h>
#define HOJ_MAX_LIMIT -1
//C C++
int LANG_CV[CALL_ARRAY_SIZE]={0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,45,46,47,48,49,50,51,52,54,55,56,59,61,62,63,64,65,66,67,68,69,70,71,72,73,74,77,78,79,81,82,84,85,86,87,88,89,90,91,92,93,97,100,103,105,106,107,108,110,111,118,122,123,125,126,143,146,148,152,159,163,165,166,167,170,176,177,179,180,182,183,184,185,186,188,191,194,195,199,200,203,209,212,215,217,218,220,222,227,228,239,240,245,249,252,257,259,260,262,263,264,272,273,275,278,280,287,288,290,292,294,295,297,298,300,305,306,312,316,319,320,321,323,327,328,330,333,334,336,339,340,341,345,347,355,362,363,364,369,370,371,375,376,383,385,386,387,390,392,393,397,400,401,403,404,412,418,422,425,426,433,447,448,449,452,453,455,456,467,468,469,471,473,474,475,479,481,482,485,486,489,492,502,504,508,510,511,551,0};//pascal
int LANG_PV[CALL_ARRAY_SIZE] = {0,3,4,54,85,174,191,248,0};
//java
int LANG_JV[CALL_ARRAY_SIZE]={0,2,5,32,36,46,54,74,85,92,122,127,148,177,320,392,395,396,397,401,402,403,404,405,406,408,412,430,447,449,479,3,7,8,9,29,33,34,40,44,45,48,49,66,81,90,95,156,167,168,185,194,197,199,200,204,205,222,226,232,237,241,263,300,326,346,349,352,356,361,363,364,381,382,393,394,398,399,400,416,423,439,448,482,487,509,510,0};
//ruby
int LANG_RV[CALL_ARRAY_SIZE] = {0,0};
//bash
int LANG_BV[CALL_ARRAY_SIZE]={0,3,4,5,6,19,20,33,45,54,63,64,65,78,122,125,140,174,175,183,191,192,195,197,199,200,201,202,221,248,0};
//python
int LANG_YV[CALL_ARRAY_SIZE]={0,1,2,6,8,11,15,20,21,22,24,30,31,32,36,37,39,40,41,44,48,49,51,54,56,60,61,63,65,68,71,72,73,74,76,81,83,86,87,88,91,92,94,97,98,101,102,109,111,114,115,120,122,124,126,131,133,135,136,140,144,146,147,148,151,155,156,161,162,165,168,171,172,173,177,181,182,183,184,185,187,191,192,194,197,198,201,208,209,211,212,213,214,215,220,222,228,231,234,238,240,241,242,244,247,251,252,253,255,256,257,260,261,265,266,267,268,273,274,276,278,282,284,285,287,292,293,294,295,298,303,308,309,311,313,316,321,322,329,332,335,337,340,342,343,344,345,346,348,351,352,353,355,356,360,366,371,372,373,377,379,384,385,388,392,393,396,398,401,403,409,413,414,419,422,425,426,427,432,433,435,436,442,443,444,445,446,448,450,451,452,453,454,456,460,461,466,468,471,472,473,474,477,480,481,483,484,485,488,493,494,496,498,499,500,502,503,510,0};
//int LANG_YV[CALL_ARRAY_SIZE]={0,6,11,22,24,32,65,86,111,122,126,133,135,165,211,222,228,234,238,265,276,287,311,322,329,335,345,377,422,433,435,444,445,477,496,0};
//php
int LANG_PHV[CALL_ARRAY_SIZE] = {0,0};
//perl
int LANG_PLV[CALL_ARRAY_SIZE] = {0,0};
//c-sharp
int LANG_CSV[CALL_ARRAY_SIZE]={0,3,5,6,19,33,45,122,125,174,175,191,192,195,197,256,338,0};
//objective-c
int LANG_OV[CALL_ARRAY_SIZE] = {0,0};
//freebasic
int LANG_BASICV[CALL_ARRAY_SIZE] = {0,0};
//scheme
int LANG_SV[CALL_ARRAY_SIZE] = {0,0};
//lua
int LANG_LUAV[CALL_ARRAY_SIZE]={0,0};
//nodejs
int LANG_JSV[CALL_ARRAY_SIZE]={0,0};
//go-lang
int LANG_GOV[CALL_ARRAY_SIZE]={0,0};
//sqlite3
int LANG_SQLV[CALL_ARRAY_SIZE]={0,1,2,3,4,5,6,7,12,13,16,17,18,20,21,22,25,28,32,36,40,41,44,49,51,52,57,64,70,85,90,92,100,119,122,127,148,152,167,168,174,177,197,216,272,320,360,392,393,395,396,397,398,400,401,402,404,405,406,407,408,409,412,430,432,433,459,461,462,465,468,469,477,487,492,497,0};
//fortran
int LANG_FV[CALL_ARRAY_SIZE]={0,2,3,4,6,85,122,392,393,397,404,405,412,453,479,0};
//matlab octave
int LANG_MV[CALL_ARRAY_SIZE]={0,1,2,3,4,5,6,9,8,10,11,12,13,16,17,18,22,24,25,28,32,33,36,40,42,48,55,56,57,60,61,64,66,72,74,76,80,83,85,88,92,96,104,111,112,116,120,122,124,126,128,131,132,133,134,135,136,144,147,148,152,157,160,164,168,169,176,179,184,185,193,200,208,215,216,224,245,248,251,256,257,264,272,279,287,288,296,304,312,315,318,320,323,328,336,337,339,344,347,349,352,368,371,372,376,377,384,385,392,393,394,395,396,397,399,400,401,402,403,404,405,406,407,408,411,412,424,431,432,433,435,436,439,440,443,446,447,448,451,452,453,455,456,462,464,468,469,472,487,488,490,492,496,497,504,510,511,0};
//Cobol
int LANG_CBV[CALL_ARRAY_SIZE]={0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,40,41,42,43,45,46,47,48,49,50,51,52,54,55,56,59,61,62,63,64,65,66,68,69,70,71,72,73,74,78,81,82,84,85,88,89,90,91,92,93,97,100,103,105,108,110,118,122,125,126,143,146,148,152,159,163,165,166,167,170,176,177,179,180,182,183,184,185,186,188,191,194,195,199,209,212,215,217,218,220,222,228,239,240,245,249,252,257,259,260,262,263,264,272,273,275,278,280,287,288,290,294,295,297,298,300,305,306,312,316,319,320,321,327,328,330,333,334,336,341,345,355,362,363,369,370,371,375,376,383,387,390,392,393,397,400,401,403,404,412,418,422,425,426,433,447,449,452,453,455,456,467,468,469,471,473,474,479,481,482,485,486,489,492,502,504,508,510,511,0};
//R
int LANG_RLV[CALL_ARRAY_SIZE]={0,11,0};
//Scratch3
int LANG_SB3V[CALL_ARRAY_SIZE]={0,11,0};

855
core/judged/judged.cc Normal file
View File

@@ -0,0 +1,855 @@
/*
* Copyright 2008 sempr <iamsempr@gmail.com>
*
* Refacted and modified by zhblue<newsclan@gmail.com>
* Bug report email newsclan@gmail.com
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
#include <time.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdlib.h>
#include <unistd.h>
#include <syslog.h>
#include <errno.h>
#include <fcntl.h>
#include <stdarg.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <signal.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#ifdef OJ_USE_MYSQL
#include <mysql/mysql.h>
#endif
#define BUFFER_SIZE 1024
#define LOCKFILE "/var/run/judged.pid"
#define LOCKMODE (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)
#define STD_MB 1048576LL
#define OJ_WT0 0
#define OJ_WT1 1
#define OJ_CI 2
#define OJ_RI 3
#define OJ_AC 4
#define OJ_PE 5
#define OJ_WA 6
#define OJ_TL 7
#define OJ_ML 8
#define OJ_OL 9
#define OJ_RE 10
#define OJ_CE 11
#define OJ_CO 12
static char lock_file[BUFFER_SIZE+32]=LOCKFILE;
static char host_name[BUFFER_SIZE];
static char user_name[BUFFER_SIZE];
static char password[BUFFER_SIZE];
static char db_name[BUFFER_SIZE];
static char oj_home[BUFFER_SIZE];
static char oj_lang_set[BUFFER_SIZE];
static int port_number;
static int max_running;
static int sleep_time;
static int sleep_tmp;
static int oj_tot;
static int oj_mod;
static int http_judge = 0;
static char http_baseurl[BUFFER_SIZE];
static char http_apipath[BUFFER_SIZE];
static char http_loginpath[BUFFER_SIZE];
static char http_username[BUFFER_SIZE];
static char http_password[BUFFER_SIZE];
static int prefetch=80;
static int oj_udp = 0;
static char oj_udpserver[BUFFER_SIZE];
static int oj_udpport=1536;
static int oj_udp_fd;
static int oj_redis = 0;
static char oj_redisserver[BUFFER_SIZE];
static int oj_redisport;
static char oj_redisauth[BUFFER_SIZE];
static char oj_redisqname[BUFFER_SIZE];
static int turbo_mode = 0;
static int use_docker = 0;
static char docker_path[BUFFER_SIZE];
static int internal_client = 1;
static int oj_dedicated=0;
static bool STOP = false;
static int DEBUG = 0;
static int ONCE = 0;
#ifdef _mysql_h
static MYSQL *conn;
static MYSQL_RES *res;
static MYSQL_ROW row;
//static FILE *fp_log;
static char query[BUFFER_SIZE*4];
#endif
void wait_udp_msg(int fd)
{
char buf[BUFFER_SIZE]; //......1024..
socklen_t len;
int count;
struct sockaddr_in clent_addr; //clent_addr............
memset(buf, 0, BUFFER_SIZE);
len = sizeof(clent_addr);
count = recvfrom(fd, buf, BUFFER_SIZE, 0, (struct sockaddr*)&clent_addr, &len); //recvfrom...............
if(count == -1)
{
printf("recieve data fail!\n");
return;
}
printf("udp client:%s\n",buf); //..client......
memset(buf, 0, BUFFER_SIZE);
// sprintf(buf, "I have recieved %d bytes data!\n", count); //..client
// printf("server:%s\n",buf); //..........
// sendto(fd, buf, BUFF_LEN, 0, (struct sockaddr*)&clent_addr, len); //.....client......clent_addr.....
}
void call_for_exit(int s) {
if(DEBUG){
STOP = true;
printf("Stopping judged...\n");
}else{
printf("HUSTOJ Refusing to stop...\n Please use kill -9 !\n");
}
}
void write_log(const char *fmt, ...) {
va_list ap;
char buffer[4096];
// time_t t = time(NULL);
// int l;
sprintf(buffer, "%s/log/client.log", oj_home);
FILE *fp = fopen(buffer, "ae+");
if (fp == NULL) {
fprintf(stderr, "openfile error!\n");
system("pwd");
}
va_start(ap, fmt);
vsprintf(buffer, fmt, ap);
fprintf(fp, "%s\n", buffer);
if (DEBUG)
printf("%s\n", buffer);
va_end(ap);
fclose(fp);
}
int after_equal(char * c) {
int i = 0;
for (; c[i] != '\0' && c[i] != '='; i++)
;
return ++i;
}
void trim(char * c) {
char buf[BUFFER_SIZE];
char * start, *end;
strcpy(buf, c);
start = buf;
while (isspace(*start))
start++;
end = start;
while (!isspace(*end))
end++;
*end = '\0';
strcpy(c, start);
}
bool read_buf(char * buf, const char * key, char * value) {
if (strncmp(buf, key, strlen(key)) == 0) {
strcpy(value, buf + after_equal(buf));
trim(value);
if (DEBUG)
printf("%s\n", value);
return 1;
}
return 0;
}
void read_int(char * buf, const char * key, int * value) {
char buf2[BUFFER_SIZE];
if (read_buf(buf, key, buf2))
sscanf(buf2, "%d", value);
}
// read the configue file
void init_judge_conf() {
FILE *fp = NULL;
char buf[BUFFER_SIZE];
host_name[0] = 0;
user_name[0] = 0;
password[0] = 0;
db_name[0] = 0;
port_number = 3306;
max_running = 3;
sleep_time = 1;
oj_tot = 1;
oj_mod = 0;
strcpy(oj_lang_set, "0,1,3,6");
strcpy(oj_udpserver, "127.0.0.1");
strcpy(docker_path, "/usr/bin/docker");
fp = fopen("./etc/judge.conf", "r");
if (fp != NULL) {
while (fgets(buf, BUFFER_SIZE - 1, fp)) {
read_buf(buf, "OJ_HOST_NAME", host_name);
read_buf(buf, "OJ_USER_NAME", user_name);
read_buf(buf, "OJ_PASSWORD", password);
read_buf(buf, "OJ_DB_NAME", db_name);
read_int(buf, "OJ_PORT_NUMBER", &port_number);
read_int(buf, "OJ_RUNNING", &max_running);
read_int(buf, "OJ_SLEEP_TIME", &sleep_time);
read_int(buf, "OJ_TOTAL", &oj_tot);
read_int(buf, "OJ_DEDICATED", &oj_dedicated);
read_int(buf, "OJ_MOD", &oj_mod);
read_int(buf, "OJ_HTTP_JUDGE", &http_judge);
read_buf(buf, "OJ_HTTP_BASEURL", http_baseurl);
read_buf(buf, "OJ_HTTP_APIPATH", http_apipath);
read_buf(buf, "OJ_HTTP_LOGINPATH", http_loginpath);
read_buf(buf, "OJ_HTTP_USERNAME", http_username);
read_buf(buf, "OJ_HTTP_PASSWORD", http_password);
read_buf(buf, "OJ_LANG_SET", oj_lang_set);
read_int(buf, "OJ_UDP_ENABLE", &oj_udp);
read_buf(buf, "OJ_UDP_SERVER", oj_udpserver);
read_int(buf, "OJ_UDP_PORT", &oj_udpport);
read_int(buf, "OJ_REDISENABLE", &oj_redis);
read_buf(buf, "OJ_REDISSERVER", oj_redisserver);
read_int(buf, "OJ_REDISPORT", &oj_redisport);
read_buf(buf, "OJ_REDISAUTH", oj_redisauth);
read_buf(buf, "OJ_REDISQNAME", oj_redisqname);
read_int(buf, "OJ_TURBO_MODE", &turbo_mode);
read_int(buf, "OJ_USE_DOCKER", &use_docker);
read_int(buf, "OJ_INTERNAL_CLIENT", &internal_client);
read_buf(buf, "OJ_DOCKER_PATH", docker_path);
}
#ifdef _mysql_h
if (oj_tot==1){
sprintf(query,
"SELECT solution_id FROM solution WHERE language in (%s) and result<2 ORDER BY result, solution_id limit %d",
oj_lang_set, prefetch * max_running );
}else{
sprintf(query,
"SELECT solution_id FROM solution WHERE language in (%s) and result<2 and MOD(solution_id,%d)=%d ORDER BY result, solution_id ASC limit %d",
oj_lang_set, oj_tot, oj_mod, prefetch * max_running );
}
#endif
sleep_tmp = sleep_time;
fclose(fp);
}
}
char * follow_link(char * path,char * buffer,int max_size){
struct stat info;
if (stat(path, &info) == -1) {
perror("stat");
return path;
}
ssize_t len = readlink(path, buffer, max_size);
if (len == -1) {
perror("readlink");
printf("Path is not a soft link !\n");
return path;
}
buffer[len] = '\0';
printf("Path is a soft link to: %s\n", buffer);
return buffer;
}
void run_client(int runid, int clientid) {
char buf[BUFFER_SIZE], runidstr[BUFFER_SIZE];
struct rlimit LIM;
LIM.rlim_max = 800;
LIM.rlim_cur = 800;
setrlimit(RLIMIT_CPU, &LIM);
LIM.rlim_max = 1024 * STD_MB;
LIM.rlim_cur = 1024 * STD_MB;
setrlimit(RLIMIT_FSIZE, &LIM);
#ifdef __mips__
LIM.rlim_max = STD_MB << 12;
LIM.rlim_cur = STD_MB << 12;
#endif
#ifdef __arm__
LIM.rlim_max = STD_MB << 11;
LIM.rlim_cur = STD_MB << 11;
#endif
#ifdef __aarch64__
LIM.rlim_max = STD_MB << 15;
LIM.rlim_cur = STD_MB << 15;
#endif
#ifdef __i386
LIM.rlim_max = STD_MB << 11;
LIM.rlim_cur = STD_MB << 11;
#endif
#ifdef __x86_64__
LIM.rlim_max = STD_MB << 15;
LIM.rlim_cur = STD_MB << 15;
#endif
setrlimit(RLIMIT_AS, &LIM);
LIM.rlim_cur = LIM.rlim_max = 800* max_running;
setrlimit(RLIMIT_NPROC, &LIM);
//buf[0]=clientid+'0'; buf[1]=0;
sprintf(runidstr, "%d", runid);
sprintf(buf, "%d", clientid);
//write_log("sid=%s\tclient=%s\toj_home=%s\n",runidstr,buf,oj_home);
//sprintf(err,"%s/run%d/error.out",oj_home,clientid);
//freopen(err,"a+",stderr);
// char * const envp[]={(char * const )"PYTHONIOENCODING=utf-8",
// (char * const )"LANG=zh_CN.UTF-8",
// (char * const )"LANGUAGE=zh_CN.UTF-8",
// (char * const )"LC_ALL=zh_CN.UTF-8",NULL};
//if (!DEBUG)
if(use_docker){
char docker_v[BUFFER_SIZE*3];
char data_v[BUFFER_SIZE*3];
char client_path[BUFFER_SIZE];
char data_path[BUFFER_SIZE*2];
char real_data_path[BUFFER_SIZE*2];
sprintf(docker_v,"%s:/home/judge",oj_home);
if(internal_client)
sprintf(client_path,"/usr/bin/judge_client");
else
sprintf(client_path,"/home/judge/src/core/judge_client/judge_client");
sprintf(data_path,"%s/data",oj_home);
char *follow=follow_link(data_path,real_data_path,sizeof(real_data_path)-1);
sprintf(data_v,"%s:/home/judge/data",follow);
if(follow!=data_path) {
printf("data volume param :%s \n",data_v);
execl(docker_path,docker_path, "container","run" ,"--pids-limit", "100","--rm","--cap-add","SYS_PTRACE", "--net=host",
"-v", docker_v,"-v",data_v, "hustoj", client_path, runidstr, buf, (char *) NULL);
}else{
execl(docker_path,docker_path, "container","run" ,"--pids-limit", "100","--rm","--cap-add","SYS_PTRACE", "--net=host",
"-v", docker_v, "hustoj", client_path, runidstr, buf, (char *) NULL);
}
}else{
execl("/usr/bin/judge_client", "/usr/bin/judge_client", runidstr, buf,
oj_home, (char *) NULL);
}
//else
// execl("/usr/bin/judge_client", "/usr/bin/judge_client", runidstr, buf,
// oj_home, "debug", (char *) NULL);
if(use_docker){
printf("DOCKER IS DOWN!\n");
}
//exit(0);
}
#ifdef _mysql_h
int executesql(const char * sql) {
if (mysql_real_query(conn, sql, strlen(sql))) {
if (DEBUG)
write_log("%s", mysql_error(conn));
sleep(2);
conn = NULL;
return 1;
} else
return 0;
}
#endif
#ifdef _mysql_h
int init_mysql() {
if (conn == NULL) {
conn = mysql_init(NULL); // init the database connection
/* connect the database */
const char timeout = 30;
mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout);
if (!mysql_real_connect(conn, host_name, user_name, password, db_name,
port_number, 0, 0)) {
if (DEBUG)
write_log("%s", mysql_error(conn));
sleep(2);
return 1;
} else {
return executesql("set names utf8");
}
} else {
return executesql("commit");
}
}
#endif
FILE * read_cmd_output(const char * fmt, ...) {
char cmd[BUFFER_SIZE*2];
FILE * ret = NULL;
va_list ap;
va_start(ap, fmt);
vsprintf(cmd, fmt, ap);
va_end(ap);
//if(DEBUG) printf("%s\n",cmd);
ret = popen(cmd, "r");
return ret;
}
int read_int_http(FILE * f) {
char buf[BUFFER_SIZE];
fgets(buf, BUFFER_SIZE - 1, f);
return atoi(buf);
}
bool check_login() {
const char * cmd =
"wget --post-data=\"checklogin=1\" --load-cookies=cookie --save-cookies=cookie --keep-session-cookies -q -O - \"%s%s\"";
int ret = 0;
FILE * fjobs = read_cmd_output(cmd, http_baseurl, http_apipath);
ret = read_int_http(fjobs);
pclose(fjobs);
return ret > 0;
}
void login() {
if (!check_login()) {
char cmd[BUFFER_SIZE*5];
sprintf(cmd,
"wget --post-data=\"user_id=%s&password=%s\" --load-cookies=cookie --save-cookies=cookie --keep-session-cookies -q -O - \"%s%s\"",
http_username, http_password, http_baseurl, http_loginpath);
system(cmd);
}
}
int _get_jobs_http(int * jobs) {
login();
int ret = 0;
int i = 0;
char buf[BUFFER_SIZE];
const char * cmd =
"wget --post-data=\"getpending=1&oj_lang_set=%s&max_running=%d\" --load-cookies=cookie --save-cookies=cookie --keep-session-cookies -q -O - \"%s%s\"";
FILE * fjobs = read_cmd_output(cmd, oj_lang_set, max_running, http_baseurl, http_apipath);
while (fscanf(fjobs, "%s", buf) != EOF) {
//puts(buf);
int sid = atoi(buf);
if (sid > 0)
jobs[i++] = sid;
//i++;
}
pclose(fjobs);
ret = i;
while (i <= max_running * prefetch)
jobs[i++] = 0;
return ret;
}
#ifdef _mysql_h
int _get_jobs_mysql(int * jobs) {
if (mysql_real_query(conn, query, strlen(query))) {
if (DEBUG)
write_log("%s", mysql_error(conn));
sleep(2);
return 0;
}
res = mysql_store_result(conn);
int i = 0;
int ret = 0;
while (res!=NULL && (row = mysql_fetch_row(res)) != NULL) {
jobs[i++] = atoi(row[0]);
}
if(res!=NULL&&!executesql("commit")){
mysql_free_result(res); // free the memory
res=NULL;
}
else i=0;
ret = i;
while (i <= max_running * prefetch )
jobs[i++] = 0;
return ret;
}
#endif
int _get_jobs_redis(int * jobs){
int ret=0;
const char * cmd="redis-cli -h %s -p %d -a %s --raw rpop %s";
while(ret<=max_running){
FILE * fjobs = read_cmd_output(cmd,oj_redisserver,oj_redisport,oj_redisauth,oj_redisqname);
if(fscanf(fjobs,"%d",&jobs[ret])==1){
ret++;
pclose(fjobs);
}else{
pclose(fjobs);
break;
}
}
int i=ret;
while (i <= max_running * prefetch )
jobs[i++] = 0;
if(DEBUG) printf("redis return %d jobs",ret);
return ret;
}
int get_jobs(int * jobs) {
if (http_judge) {
return _get_jobs_http(jobs);
} else {
if(oj_redis){
return _get_jobs_redis(jobs);
}else{
#ifdef _mysql_h
return _get_jobs_mysql(jobs);
#else
return 0;
#endif
}
}
}
#ifdef _mysql_h
bool _check_out_mysql(int solution_id, int result) {
char sql[BUFFER_SIZE];
sprintf(sql,
"UPDATE solution SET result=%d,time=0,memory=0,judgetime=NOW() WHERE solution_id=%d and result<2 LIMIT 1",
result, solution_id);
if (mysql_real_query(conn, sql, strlen(sql))) {
syslog(LOG_ERR | LOG_DAEMON, "%s", mysql_error(conn));
return false;
} else {
if (conn!=NULL&&mysql_affected_rows(conn) > 0ul)
return true;
else
return false;
}
}
#endif
bool _check_out_http(int solution_id, int result) {
login();
const char * cmd =
"wget --post-data=\"checkout=1&sid=%d&result=%d\" --load-cookies=cookie --save-cookies=cookie --keep-session-cookies -q -O - \"%s%s\"";
int ret = 0;
FILE * fjobs = read_cmd_output(cmd, solution_id, result, http_baseurl, http_apipath);
fscanf(fjobs, "%d", &ret);
pclose(fjobs);
return ret;
}
bool check_out(int solution_id, int result) {
if(oj_redis||oj_tot>1) return true;
if (http_judge) {
return _check_out_http(solution_id, result);
} else{
#ifdef _mysql_h
return _check_out_mysql(solution_id, result);
#else
return 0;
#endif
}
}
static int workcnt = 0;
int work() {
// char buf[1024];
static int error=0; // 出错计数器
int retcnt = 0;
int i = 0;
static pid_t ID[100];
int runid = 0;
int jobs[max_running * prefetch + 1];
pid_t tmp_pid = 0;
//for(i=0;i<max_running;i++){
// ID[i]=0;
//}
for(i=0;i<max_running *prefetch +1 ;i++)
jobs[i]=0;
//sleep_time=sleep_tmp;
/* get the database info */
if (!get_jobs(jobs)){
return 0;
}
/* exec the submit */
for (int j = 0; jobs[j] > 0; j++) {
runid = jobs[j];
if (runid % oj_tot != oj_mod)
continue;
if (workcnt >= max_running) { // if no more client can running
tmp_pid = waitpid(-1, NULL, WNOHANG); // wait 4 one child exit
if (DEBUG) printf("try get one tmp_pid=%d\n",tmp_pid);
for (i = 0; i < max_running; i++){ // get the client id
if (ID[i] == tmp_pid){
workcnt--;
retcnt++;
ID[i] = 0;
break; // got the client id
}
}
if(use_docker && (i==max_running || ID[i]!=0)) error++; else error=0; // check if docker service is hanged up
} else { // have free client
for (i = 0; i < max_running; i++) // find the client id
if (ID[i] == 0){
break; // got the client id
}
}
if(i<max_running){
if (workcnt < max_running && check_out(runid, OJ_CI)) {
workcnt++;
ID[i] = fork(); // start to fork
if (ID[i] == 0) {
if (DEBUG){
write_log("Judging solution %d", runid);
write_log("<<=sid=%d===clientid=%d==>>\n", runid, i);
}
run_client(runid, i); // if the process is the son, run it
workcnt--;
exit(0);
}
} else {
// ID[i] = 0;
if(DEBUG){
if(workcnt<max_running)
printf("check out failure ! runid:%d pid:%d \n",i,ID[i]);
else
printf("workcnt:%d max_running:%d ! \n",workcnt,max_running);
}
usleep(5000);
}
}
if(DEBUG)
printf("workcnt:%d max_running:%d ! \n",workcnt,max_running);
if(use_docker && (error>= max_running*prefetch )){ // reboot docker
if(DEBUG) printf("---------------------------------------------restarting--------------------------------------\n");
//system("/usr/sbin/service docker restart");
#ifdef _mysql_h
executesql("update solution set result=1 where result >1 and result <4 ");
#endif
sleep(1);
error=0;
}
}
int NOHANG=0;
if(( max_running >3 || oj_dedicated ) && ( max_running > 7 || rand()%100>20 ) ) NOHANG=WNOHANG; // CPU小于4个 占用大约80%左右，不要打满；大于7个，瓶颈转为IO，全力工作
while ((tmp_pid = waitpid(-1, NULL, NOHANG )) > 0) { // if run dedicated judge using WNOHANG
for (i = 0; i < max_running; i++){ // get the client id
if (ID[i] == tmp_pid){
workcnt--;
retcnt++;
ID[i] = 0;
break; // got the client id
}
}
printf("tmp_pid = %d\n", tmp_pid);
}
if (!http_judge) {
#ifdef _mysql_h
if(res!=NULL) {
mysql_free_result(res); // free the memory
res=NULL;
}
executesql("commit");
#endif
}
if (DEBUG && retcnt)
write_log("<<%ddone!>>", retcnt);
//free(ID);
//free(jobs);
return retcnt;
}
int lockfile(int fd) {
struct flock fl;
fl.l_type = F_WRLCK;
fl.l_start = 0;
fl.l_whence = SEEK_SET;
fl.l_len = 0;
return (fcntl(fd, F_SETLK, &fl));
}
int already_running() {
int fd;
char buf[16];
fd = open(lock_file, O_RDWR | O_CREAT, LOCKMODE);
if (fd < 0) {
syslog(LOG_ERR | LOG_DAEMON, "can't open %s: %s", LOCKFILE,
strerror(errno));
exit(1);
}
if (lockfile(fd) < 0) {
if (errno == EACCES || errno == EAGAIN) {
close(fd);
return 1;
}
syslog(LOG_ERR | LOG_DAEMON, "can't lock %s: %s", LOCKFILE,
strerror(errno));
exit(1);
}
ftruncate(fd, 0);
sprintf(buf, "%d", getpid());
write(fd, buf, strlen(buf) + 1);
return (0);
}
int daemon_init(void)
{
pid_t pid;
if ((pid = fork()) < 0)
return (-1);
else if (pid != 0)
exit(0); /* parent exit */
/* child continues */
setsid(); /* become session leader */
chdir(oj_home); /* change working directory */
umask(0); /* clear file mode creation mask */
close(0); /* close stdin */
close(1); /* close stdout */
close(2); /* close stderr */
int fd = open( "/dev/null", O_RDWR );
dup2( fd, 0 );
dup2( fd, 1 );
dup2( fd, 2 );
if ( fd > 2 ){
close( fd );
}
return (0);
}
void turbo_mode2(){
#ifdef _mysql_h
if(turbo_mode==2){
char sql[BUFFER_SIZE];
sprintf(sql," CALL `sync_result`();");
if (mysql_real_query(conn, sql, strlen(sql)));
}
#endif
}
int main(int argc, char** argv) {
int oj_udp_ret=0;
DEBUG = (argc > 2);
ONCE = (argc > 3);
if (argc > 1)
strcpy(oj_home, argv[1]);
else
strcpy(oj_home, "/home/judge");
chdir(oj_home); // change the dir
sprintf(lock_file,"%s/etc/judge.pid",oj_home);
if (!DEBUG)
daemon_init();
if ( already_running()) {
syslog(LOG_ERR | LOG_DAEMON,
"This daemon program is already running!\n");
printf("%s already has one judged on it!\n",oj_home);
return 1;
}
if(!DEBUG)
system("/sbin/iptables -A OUTPUT -m owner --uid-owner judge -j DROP");
// struct timespec final_sleep;
// final_sleep.tv_sec=0;
// final_sleep.tv_nsec=500000000;
init_judge_conf(); // set the database info
if(oj_udp){
oj_udp_fd = socket(AF_INET, SOCK_DGRAM, 0);
if(oj_udp_fd<0)
printf("udp fd open failed! \n");
struct sockaddr_in ser_addr;
memset(&ser_addr, 0, sizeof(ser_addr));
ser_addr.sin_family = AF_INET;
ser_addr.sin_addr.s_addr = inet_addr(oj_udpserver);
ser_addr.sin_port = htons(oj_udpport);
struct timeval timeOut;
timeOut.tv_sec = sleep_time; //..5s..
timeOut.tv_usec = 0;
if (setsockopt(oj_udp_fd, SOL_SOCKET, SO_RCVTIMEO, &timeOut, sizeof(timeOut)) < 0)
{
printf("time out setting failed\n");
}
oj_udp_ret=bind(oj_udp_fd, (struct sockaddr*)&ser_addr, sizeof(ser_addr));
if(oj_udp_ret<0)
printf("udp fd open failed! \n");
}
signal(SIGQUIT, call_for_exit);
signal(SIGINT, call_for_exit);
signal(SIGTERM, call_for_exit);
int j = 1;
int n = 0;
while (!STOP) { // start to run until call for exit
n=0;
while (j && (http_judge
#ifdef _mysql_h
|| !init_mysql()
#endif
)) {
j = work();
n+=j;
if(turbo_mode==2&&(n>max_running*10||j<max_running)){
turbo_mode2();
n=0;
}
if(ONCE && j==0) break;
}
turbo_mode2();
if(ONCE && j==0) break;
if(n==0){
printf("workcnt:%d\n",workcnt);
if(oj_udp&&oj_udp_ret==0){
if(STOP) return 1;
wait_udp_msg(oj_udp_fd);
if(DEBUG) printf("udp job ... \n");
}else{
sleep(sleep_time);
if(DEBUG) printf("sleeping ... %ds \n",sleep_time);
}
}
j = 1;
}
#ifdef _mysql_h
mysql_close(conn);
#endif
return 0;
}

212
core/judged/judgehub.cc Normal file
View File

@@ -0,0 +1,212 @@
/*
* Copyright 2021 zhblue <newsclan@gmail.com>
*
* This file is part of HUSTOJ.
*
* HUSTOJ is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* HUSTOJ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HUSTOJ. if not, see <http://www.gnu.org/licenses/>.
*/
#include <time.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdlib.h>
#include <unistd.h>
#include <syslog.h>
#include <errno.h>
#include <fcntl.h>
#include <stdarg.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <signal.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#define BUFFER_SIZE 1024
static char oj_udpserver[BUFFER_SIZE]="127.0.0.1";
static int oj_udpport=2008;
static int oj_udp_fd;
static char oj_hub_base[BUFFER_SIZE]="/home/saas";
static bool STOP = false;
static int DEBUG = 0;
void wait_udp_msg(int fd)
{
char buf[BUFFER_SIZE]; //......1024..
char dir[BUFFER_SIZE]; //......1024..
char cmd[BUFFER_SIZE<<2]; //......1024..
char cnf[BUFFER_SIZE<<2]; //......1024..
socklen_t len;
int count;
struct sockaddr_in clent_addr; //clent_addr............
memset(buf, 0, BUFFER_SIZE);
len = sizeof(clent_addr);
count = recvfrom(fd, buf, BUFFER_SIZE, 0, (struct sockaddr*)&clent_addr, &len); //recvfrom.......
if(count == -1)
{
printf("recieve data fail!\n");
return;
}
printf("task msg:%s\n",buf); //..client......
sscanf(buf,"%s\n",dir);
sprintf(cmd,"judged %s/%s/ debug 4",oj_hub_base,dir); //..client......
sprintf(cnf,"%s/%s/etc/judge.conf",oj_hub_base,dir);
struct stat cnfstat;
if(stat(cnf,&cnfstat)){
printf("%s read fail...\n",cnf);
}else{
struct timeval now;
gettimeofday(&now,NULL);
printf(" now_time:%ld\n",now.tv_sec);
printf("cnf_mtime:%ld\n",cnfstat.st_mtime);
if(fork()==0){
printf("%s\n",cmd);
exit(system(cmd));
}else{
waitpid(-1, NULL, WNOHANG);
}
}
memset(buf, 0, BUFFER_SIZE);
// sprintf(buf, "I have recieved %d bytes data!\n", count); //..client
// printf("server:%s\n",buf); //..........
// sendto(fd, buf, BUFF_LEN, 0, (struct sockaddr*)&clent_addr, len); //.....client......clent_addr.....
}
void call_for_exit(int s) {
STOP = true;
printf("Stopping judge hub ...\n");
}
void write_log(const char *fmt, ...) {
va_list ap;
char buffer[4096];
// time_t t = time(NULL);
// int l;
sprintf(buffer, "/var/log/judgehub.log");
FILE *fp = fopen(buffer, "ae+");
if (fp == NULL) {
fprintf(stderr, "openfile error!\n");
}
va_start(ap, fmt);
vsprintf(buffer, fmt, ap);
fprintf(fp, "%s\n", buffer);
if (DEBUG)
printf("%s\n", buffer);
va_end(ap);
fclose(fp);
}
int lockfile(int fd) {
struct flock fl;
fl.l_type = F_WRLCK;
fl.l_start = 0;
fl.l_whence = SEEK_SET;
fl.l_len = 0;
return (fcntl(fd, F_SETLK, &fl));
}
int daemon_init(void)
{
pid_t pid;
if ((pid = fork()) < 0)
return (-1);
else if (pid != 0)
exit(0); /* parent exit */
/* child continues */
setsid(); /* become session leader */
umask(0); /* clear file mode creation mask */
close(0); /* close stdin */
close(1); /* close stdout */
close(2); /* close stderr */
int fd = open( "/dev/null", O_RDWR );
dup2( fd, 0 );
dup2( fd, 1 );
dup2( fd, 2 );
if ( fd > 2 ){
close( fd );
}
return (0);
}
void turbo_mode2(){
#ifdef _mysql_h
if(turbo_mode==2){
char sql[BUFFER_SIZE];
sprintf(sql," CALL `sync_result`();");
if (mysql_real_query(conn, sql, strlen(sql)));
}
#endif
}
int main(int argc, char** argv) {
int oj_udp_ret=0;
DEBUG = argc>3 ;
if(argc >1){
strcpy(oj_hub_base,argv[1]);
}else{
printf("check out https://github.com/zhblue/hustoj/blob/master/wiki/JudgeHub.md \n");
}
if(argc >2){
sscanf(argv[2],"%d",&oj_udpport);
}
if (!DEBUG)
daemon_init();
oj_udp_fd = socket(AF_INET, SOCK_DGRAM, 0);
if(oj_udp_fd<0){
printf("udp fd open failed! \n");
exit(-1);
}
struct sockaddr_in ser_addr;
memset(&ser_addr, 0, sizeof(ser_addr));
ser_addr.sin_family = AF_INET;
ser_addr.sin_addr.s_addr = inet_addr(oj_udpserver);
ser_addr.sin_port = htons(oj_udpport);
struct timeval timeOut;
timeOut.tv_sec = 60 ; //..5s..
timeOut.tv_usec = 0;
if (setsockopt(oj_udp_fd, SOL_SOCKET, SO_RCVTIMEO, &timeOut, sizeof(timeOut)) < 0)
{
printf("time out setting failed\n");
}
oj_udp_ret=bind(oj_udp_fd, (struct sockaddr*)&ser_addr, sizeof(ser_addr));
if(oj_udp_ret<0)
printf("udp fd open failed! \n");
//signal(SIGQUIT, call_for_exit);
//signal(SIGINT, call_for_exit);
//signal(SIGTERM, call_for_exit);
while (!STOP) { // start to run until call for exit
printf("waiting task:%d\n",oj_udpport);
if(STOP) return 100;
wait_udp_msg(oj_udp_fd);
if(DEBUG) printf("udp job ... \n");
waitpid(-1, NULL, WNOHANG);
}
return 0;
}

10
core/judged/makefile Normal file
View File

@@ -0,0 +1,10 @@
all: judged.cc judged.http judgehub
g++ -Wall -c -DOJ_USE_MYSQL -I/usr/local/mysql/include/mysql -I/usr/include/mysql judged.cc
g++ -Wall -o judged judged.o -L/usr/local/mysql/lib/mysql -L/usr/lib/mysql -L/usr/lib64/mysql -lmysqlclient
judged.http: judged.cc
g++ -Wall -o judged.http judged.cc
judgedhub: judgehub.cc
g++ -Wall -o judgehub judgehub.cc

24
core/make.sh Executable file
View File

@@ -0,0 +1,24 @@
#!/bin/bash
cd judged
make
chmod +x judged
cp judged /usr/bin
cd ../judge_client
make
chmod +x judge_client
cp judge_client /usr/bin
cd ../sim/sim_3_01
make fresh
make exes
chmod +x sim*
cp sim_c.exe /usr/bin/sim_c
cp sim_c++.exe /usr/bin/sim_cc
cp sim_java.exe /usr/bin/sim_java
cp sim_pasc.exe /usr/bin/sim_pas
cp sim_text.exe /usr/bin/sim_text
cp sim_lisp.exe /usr/bin/sim_scm
cd ..
cp sim.sh /usr/bin
chmod +x /usr/bin/sim.sh
#ln -fs /usr/bin/sim_c /usr/bin/sim_cc 2>&1 > /dev/null
echo "done!"

27
core/sim/sim.sh Executable file
View File

@@ -0,0 +1,27 @@
#!/bin/bash
umask 0077
EXTENSION=`echo "$1" | cut -d'.' -f2`
FIRST=""
for i in `ls ../data/$2/ac/*.$EXTENSION`
do
echo "i:$i"
if [ ! -e "/usr/bin/sim_$EXTENSION" ]
then
EXTENSION="text";
fi
sim=`/usr/bin/sim_$EXTENSION -p $1 $i |grep ^$1|awk '{print $4}'`
if [ ! -z "$sim" ] && [ "$sim" -gt 80 ]
then
sim_s_id=`basename $i`
echo "$sim $sim_s_id" >sim
exit $sim
fi
FIRST="false"
done
if [ -z "$FIRST" ] ;then
echo "first answer"
else
echo $FIRST
fi
echo "0 0" > sim
exit 0;

756
core/sim/sim_3_01/ChangeLog Normal file
View File

@@ -0,0 +1,756 @@
2017-03-19 Dick Grune <Gebruiker@GEBRUIKER-PC>
* c++lang.l contributed by Evin Murphy (evin.murphy@ucdconnect.ie).
2016-08-07 Dick Grune <Gebruiker@GEBRUIKER-PC>
* compare.c (lcs): Finding the text into which i1 points was done by
linear search. Replaced by binary search. This resulted in no direct
improvement but should be beneficial for very large numbers of files.
2016-08-05 Dick Grune
* hash.c (make_forward_references_perfect): This routine now
constructs perfect forward references. This allows the comparison code
in compare.c (lcs) to skip the first Min_Run_Size tokens when
comparing two chunks. This yields about 8% speed-up.
2016-08-03 Dick Grune
* hash.c (make_forward_references_using_hash): A lot of work was being
done to compute the hash value of a run, using sampling etc. A new
scheme was implemented in which the hash value consists of all tokens
in the run, each circularly left-shifted over a distance proportional
to its position in the run. This value can be computed incrementally
in constant time, by removing the oldest and adding the newest token
at each position.
Care had to be taken not to produce negative values for entities of
type size_t.
For moderately large comparisons (~ 10M tokens) this yields an
over-all speed-up of about 20%.
2016-05-29 Dick Grune
* pass3.c: Surprisingly, very large runs of the program can get out of
memory far into the computation, in spite of the fact that sim
allocates almost all its memory while reading the input. However, the
found runs of text are collected using Mallocked memory, for producing
sorted output. In many cases one would rather have unsorted output
than no output at all. Decided to implement the -u option, for
unsorted output, thus not using any additional memory.
2016-05-13 Dick Grune
* compare.c (lcs): the -X option works. It uses the linear algorithm
to compare a file F to two segments of the text: the segment from the
start of the text to the start of F, and from the end of F to the end
of the text. Runs found are then reported in percentages.
2016-05-06 Dick Grune
* compare.c: The results of the analysis in
Similarity_Percentage_Computation.tex have been implemented, and under
the -p option percentages are now computed consistently. In
particular, the files in the directory Contributors/Debora_Weber-Wulff/
yield the same relative percentages independent of the order they are
entered in; see Pfubar_shuffle and Pslash_shuffle. Note: both parts of
Pslash_shuffle were shuffled independently.
2016-04-10 Dick Grune
* runs.c: aiso (arbitrary in sorted out) package replaced by an
instantiation of the package sortlist.
2014-02-17 Dick Grune
* sim.c (is_new_old_separator): MinGW sometimes (?) interprets the /
as a command-line argument as a reference to the MinGW tree, which
makes the / unusable as a separator. Even escaping it ("/") does not
help. Added the | as a separator.
2014-01-26 Dick Grune
* %z from Marcus Brinkmann implemented by a routine size_t2string.
2014-01-26 Dick Grune
* Sizes on my present machine (HP 6730b laptop):
unsigned short int: 2
unsigned int: 4
unsigned long int: 4
unsigned long long int: 8
2013-05-31 Dick Grune
* (hash.c) Better hash2() function.
2013-04-28 Dick Grune
* Markus Brinkmann (marcus.brinkmann@ruhr-uni-bochum.de) supplied a
64-bit version. %z does not work on MinGW C.
2012-11-28 Dick Grune
* newargs.c (recursive_args): Liqun Chen (liqun.chen@hp.com)
submitted a bug report noting that the separator / is expanded under
the -R option. Corrected.
2012-09-30 Dick Grune
* pass2.c (pass2_txt): Boyd Blackwell (Boyd.Blackwell@anu.edu.au)
submitted a bug report in which the line numbers (and runs
representations) were way off (75 lines). The input files were
characterized by extremely long lines, hundreds of tokens (max. 521).
After 2.5 days of debugging the cause was found: 1. since the mapping
from token positions to line numbers is stored as the difference of
the token positions from one line to the next (see text.c); 2. since
these differences are stored in unsigned chars to save space; 3. since
the nl_buff mechanism is switched off when one of these unsigned
characters overflow; and since 521 tokens on one line overflowed this
unsigned char, the nl_buff mechanism was shut off.
Since when there is no nl_buff information in pass2, pass2 resorts to
rereading the input file calling yylex again; 2. since the preceding
file had few runs to find line number to, the preceding file was not
read to the end, and the rest remained in flex's buffer, so a portion
of the preceding file seemed prefixed to the present file, adding 75
lines to it.
Remedy: flushing flex's buffer explicitly in pass2_txt(); this is
simpler than using flex's YY_BUFFER_STATE mechanism.
Advice: get rid of the nl_buff mechanism; it is no longer relevant.
2012-06-09 Dick Grune
* lang.h:
The *lang.l files are unusual in two respects:
1. they present two interfaces to the rest of the system:
language.[ch], static data about the language, and lang.[ch], dynamic
data about the input file's content;
2. both interfaces come with multiple implementations, one for each
*lang.l file; i.e., they are abstract.
This has been sorted out with some difficulty.
2012-05-08 Dick Grune
* Changed to 16-bit tokens, for better resolution for sim_text and
on -F option, and for UTF-8 input.
It was not worth while to save the 8-bit token code: on serious
comparisons the increase in memory usage is about 10% (330 000 on a
maximum allocation of 3 030 976 for comparing the sources of MCD2).
2009-03-11 Dick Grune <dick@flits.few.vu.nl>
* newargs.c: added -R option to follow directories recursively.
See recursive_args().
2008-09-22 <Dick@ACER>
* added newargs.[ch], to supply file names from standard input,
for those compilers that do not have the @ facility. Implemented
without fixed limits.
2008-09-21 <Dick@ACER>
* changed default format back to original, and inverted the
-v(erbose) option into a -T(erse) option.
2008-03-31 Dick Grune <dick@flits.few.vu.nl>
* *.l: the following are not universally recognized; removed.
%option nounput
%option never-interactive
2008-03-31 <Dick@ACER>
Introduced aiso.* and Malloc.? as imported modules.
2007-11-21 Carlos Maziero <maziero@ppgia.pucpr.br>
- output format modified in order to facilitate "grep" filtering
- added option "-v" for a more verbose output
- added option "-tN" to define a threshold %N (only similarities
over N% are shown)
- fixed SEGV on writing to the output file
- the file list can be informed through STDIN (one file per line,
accepts "/" marker); this is useful for compilers that lack the
@ facility
2007-08-23 Dick Grune <dick@hydra.cs.vu.nl>
LICENSE.txt added.
2006-11-27 Dick Grune <dick@hydra.cs.vu.nl>
Removal of setbuff() for compatibility.
2005-01-17 Dick Grune <dick@blade014.cs.vu.nl>
Corrections by Jerry James <james@eecs.ku.edu>; ANSIizing, etc.
2004-08-05 Dick Grune <dick@blade014.cs.vu.nl>
Finished the 'percentage' option.
08-Nov-2001 Dick Grune
Begun to add a 'percentage' option, which will express the
similarity between two files in percents.
27-Sep-2001 Dick Grune
Split add_run() off from compare.c into add_run.c, to accommodate
different add_run()s, for different types of processing.
27-Nov-1998 Dick Grune
Installed a Miranda version supplied by Emma Norling (ejn@cs.mu.oz.au)
23-Feb-1998 Dick Grune
Renamed text.l to textlang.l for uniformity and to make room for
a possible module text.[ch].
Isolated a module for handling the token array from buff.[ch] to
tokenarray.[ch], and renamed buff.[ch] to text.[ch].
23-Feb-1998 Dick Grune
There is probably not much point in abandoning the nl_buff list
when running out of memory for TokenArray[]: each token costs 1
byte for the token and 4 bytes for the entry in
forward_references[], a total of 5 bytes. There are about 3
tokens to a line, together requiring 15 bytes, plus 1 byte in
nl_buff yields 16 bytes. So releasing nl_buff frees only 1/16 =
6.7 % of memory.
Since the code is a bother, I removed it. Note that nl_buff is
still abandoned when the number of tokens in a line does not fit
in one unsigned char (but that is not very likely to happen).
21-Feb-1998 Dick Grune
Printing got into an infinite loop when the last line of the
input was not terminated by a newline AND contained tokens that
were included in a matching run.
This was due to a double bug: 1. the non-terminated line was not
registered properly in NextTextTokenObtained() / CloseText(),
and 2. the loop in pass 2 which sets the values of
pos->ps_nl_cnt was terminated prematurely when the file turned
out to be shorter than the list of pos-es indicated.
Both bugs were corrected, the first by supplying an extra
newline in CloseText() when one is found missing, and the second
by rewriting the list-parallel loop in pass 2.
02-Feb-1998 Dick Grune
Pascal does not differentiate between strings and characters
(strings of one character); this difference has been removed
from pascallang.l.
22-Jan-1998 Dick Grune
Detection of non-ASCII characters added. Since the lexical
analyser itself generates non-ASCII characters, the test must occur
earlier. We could replace the input routine of lex by a
checking routine, but with several lex-es going around, we want
a more lex-independent solution. To allow each language its own
restrictions about non-ASCII characters, the check is
implemented in the *lang.l files.
28-Nov-1997 Dick Grune
Changed the name of the C similarity tester 'sim' to 'sim_c', for
uniformity with sim_java, etc.
23-Nov-1997 Dick Grune
Java version finished; checked by Matty Huntjens and crew.
24-Jun-1997 Dick Grune
Started on a Java version, by copying the C version.
22-Jun-1997 Dick Grune
Modern lexical analysers, among which flex, read the entire input into
a buffer before they issue the first token. As a result, ftell() no
longer gives a usable indication of the position of a token in a file.
This pulls the rug from under the nl_buff mechanism in buff.c, which
is removed. We loose a valuable optimization this way, but there just
seems to be no way to keep it.
Note that this has nothing to do with the problem in MS-DOS of
character count and fseek position not being synchronized. That
problem has been solved on June 14, 1991 (which see) and the code has
been running OK since.
18-Jun-1997 Dick Grune
The thought has occurred to use McCreight's linear longest common
substring algorithm rather than the existing algorithm, which has a
small quadratic component. There are a couple of problems with this:
1. We need the longest >non-overlapping< common substring;
McCreight provides just the longest. It is not at all clear
how to modify the algorithm.
2. Once we have found our LCS, we want to find the
one-but-longest; it is far from obvious how to do that in
McCreight's algorithm.
3. Once we have found our LCS, we want to take one of its
copies out of the game, to suppress duplicate messages.
Again, it is difficult to see how to do that, without
redoing all the calculations.
4. McCreight's algorithm seems to require about two binary
tree nodes per token, say 8 bytes, which is double we
use now.
17-Jun-1997 Dick Grune
Did some experimenting with the hash function; it is still
pretty bad: the simple-minded second sweep through
forward_references easily removes another 80-99% of false hits.
Next, a third sweep that does a full comparison will remove another
large percentage.
So I have left in the second sweep in all cases.
There are a couple of questions here:
1. Can we find a better hash function, or will we forever need a
second sweep?
2. Does it actually matter, or will we loose on more expensive
hashing what we gain by having a better set of forward
references in compare.c?
16-Jun-1997 Dick Grune
Cleaned up sim.h and renamed aiso.[ch] to runs.[ch] since they
are instantiations of the aiso module concerned with runs.
Aiso.[spc|bdy] stays aiso.[spc|bdy], of course.
16-Jun-1997 Dick Grune
Redid largest_function() in algollike.c.
Corrected bug in CheckRun; it now always removes NonFinals from
the end, even when it has first applied largest_function().
15-Jun-1997 Dick Grune
Reorganized the layers around the input file. There were and
still are three layers: lang, stream and buff.
Since the lex_X variables are hoisted unchanged through the levels
lang, stream, and buff, to be used by pass1, pass2, etc., they
have to be placed in a module of their own.
The token-providing module 'lang' has three interfaces:
- lang.h, which provides access to the lowest-level token
routines, to be used by the next level.
- lex.h, which provides the lex variables, to be used by
all and sundry.
- language.h, which provides language-specific info about
tokens, concerning their suitability as initial
and final tokens, to be used by higher levels.
This structure is not satisfactory, but it is also unreasonable
to combine them in one interface.
There is no single lang.c; rather it is represented by the
various Xlang.c files generated from the Xlang.l files.
14-Jun-1997 Dick Grune
Added a Makefile zip entry to parallel the shar entry.
13-Jun-1997 Dick Grune
A number of simplifications, in view of better software and bigger
machines:
- Removed good_realloc from hash.c; I don't think there are
any bad reallocs left.
- Removed the option to run without forward_references.
On a 16Mb machine this means you have at least 2M tokens;
using a quadratic algorithm will take 4*10^6 sec. at an
impossible rate of 1M actions/sec., which is some 50 days.
Forget it.
- Renamed lang() to print_stream(), and incorporated it in sim.c
- Removed the MSDOS subdirectory mechanism in the Makefile.
- Removed the funny and sneaky double parameter expansion in
the call of idf_in_list().
12-Jun-1997 Dick Grune
Converted to ANSI C. Removed cport.h.
09-Jan-1995 Dick Grune
Decided not to do directories: they usually contain extraneous
files and doing sim * is simple enough anyway.
09-Sep-1994 Dick Grune
Added system.h to cater for the (few) differences between Unix and DOS.
The #define int32 is also supplied there.
05-Sep-1994 Dick Grune
Added many prototype declarations using cport.h.
Added a depend entry to the Makefile.
31-Aug-1994 Dick Grune
All these changes require a 32 bit integer; introduced a #define
int32, set from the command line in the Makefile.
25-Aug-1994 Dick Grune
It turned out that one of the most often called routines was .rem,
from idf_hashed() in idf.c. Moving the % out of the loop chafed off
another 6% and reduced the time to 18.4 sec.
19-Aug-1994 Dick Grune
With very large files (e.g., concatenated /usr/man/man1/*) the fixed
built-in hash table size of 10639 is no longer satisfactory. Hash.c
now finds a prime about 8 times smaller than the text_size to use
for hash table size; this achieves optimal speed-up without gobbling
up too much memory. Reduced the time for the above file from 30.2
sec. to 19.6 sec.
For checking, the same test was run with all hashing off; it took
20h 27m 19s = 73639 sec. But it worked.
11-Aug-1994 Dick Grune
For large values of MinRunSize (>1000) a large part of the time
(>two-thirds) was spent in calculating the hash values for each
position in the input, since the cost of this calculation was
proportional to MinRunSize. We now sample a maximum of 24 tokens
from the input string to calculate the hash value, and avoid
overflow. On my workstation, this reduces the time for
sim_text -r 1000 -n /usr/man/man1/*
from 60 sec to 21 sec.
30-Jun-1992 Dick Grune,kamer R4.40,telef. 5778
There was an amazing bug in buff.c where NextTextToken() for pass 2
omitted to set lex_token to EOL when retrieving newline info from
nl_buff. Worked until now!?!
23-Sep-1991 Dick Grune
Cport.h introduced, CONST and *.spc only.
17-Sep-1991 Dick Grune
The position-sorting routine in pass2.c has been made into a
separate generic module.
14-Jun-1991 Dick Grune (dick@cs.vu.nl) at dick.cs.vu.nl
Replaced the determination of the input position through counting
input characters by calls of ftell(); this is cleaner and the other
method will never work on MSDOS.
30-May-1989 Dick Grune (dick) at dick
Replaced the old top-100 module (which had been extended to top-10000
already anyway) by the new aiso (arbitrary-in sorted-out) module.
This caused a considerable speed-up on the Mod2 test bed:
%time cumsecs #call ms/call name
17.9 99.20 7209 13.76 _InsertTop
0.3 1.37 7209 0.19 _InsertAiso
It turns out that malloc() is not a serious problem, so no special
version for the aiso module is required.
23-May-1989 Dick Grune (dick) at dick
No more uncommented comment at the end of preprocessor lines, to
conform to ANSI C.
23-May-1989 Dick Grune (dick) at dick
Added code in the X.l files to (silently) reject characters over 0200.
This does not really help, since lex stops on null chars. Ah, well.
19-May-1989 Dick Grune (dick) at dick
Made the token as handled by sim into an abstract data type, for
aesthetic reasons. Sign extension is still a problem.
03-May-1989 Dick Grune (dick) at dick
Optimized lcs() by first checking from the end if a sufficiently long
run is present; if in fact only the first 12 tokens match, chances
are good that you can reject the run right away by first testing
the 20th token, then the 19th, and so on.
21-Apr-1989 Dick Grune (dick) at dick
A run of sim_m2 finding 7209 similarities raised the question of
the appropriateness of the linear sort in sort_pos(). Profiling
showed that in this case sorting takes all of 7.5 % of the total
time. Putting the word register in in the right places in
sort_pos() lowered this number to 4.6%.
20-Apr-1989 Dick Grune (dick) at dick
Moved the test for MayBeStartOfRun() from compare.c (where it is
done again and again) to hash.c, where its effect is incorporated in
the forward reference chain.
14-Apr-1989 Dick Grune (dick) at dick
Replaced elem_of() by bit tables, headers[] and trailers[], to be
prefilled from Headers[] and Trailers[] by a call of
InitLanguage(). This saves a few percents.
13-Apr-1989 Dick Grune (dick) at dick
Implemented the -e and the -S option, by putting yet another loop
in compare.c
13-Apr-1989 Dick Grune (dick) at dick
The -- option (displaying the tokens) will now handle more than one
file.
20-Jan-1989 Dick Grune (dick) at dick
After the modification of 19-Dec-88, 12% of the time went into
updating the positions in the chunks, as they were produced by the
matching process. This matching process identifies runs (matches)
by token position, which has to be recalculated to lseek positions
and line numbers. To this end the files are read again, and for
each line all positions found were checked to see if they applied
to this line; this was a awfully stupid algorithm, but since much
more time was spent elsewhere, it did not really matter. With all
the saving below, however, it had risen to second position, after
yylook() with 35%.
Th solution was, to sort the positions in the same order in which
they would be met by the reading of the files. The process is then
linear. This required some extensive hacking in pass2.c
06-Jan-1989 Dick Grune (dick) at dick
The modification below did indeed save 25%. The newline information
is now reduced to 2 shorts; 2 chars were not enough, since some
lines are longer that 127 bytes, and a char and a short together
take as much room as two shorts.
19-Dec-1988 Dick Grune (dick) at dick
To avoid reading the files twice (which is still taking 25% of the
time), the first pass will now collect newline information for the
second pass in a buffer called nl_buff[]. This buffer, and the
original token buffer now named TokenArray[], are managed by the file
buff.c, which implements a layer between stream.h and pass?.c. This
layer provides OpenText(), NextTextToken() and CloseText(), each
with a parameter telling which pass it is.
06-Dec-1988 Dick Grune (dick) at dick
As an introduction to removing the second pass altogether, the
first and second scan were unified, i.e., their input is identical.
This also means that the call sim -[12] has now been replaced by
one call: sim --.
23-Sep-1988 Dick Grune (dick) at dick
Dynamic allocation of line buffers in pass 3. This removes the
restriction on the page width.
22-Sep-1988 Dick Grune (dick) at dick
In order to give better messages on incorrect calls to sim, the
whole option handling has been concentrated in a file option.c and
separated from the options and their messages themselves. See sim.c
07-Sep-1988 Dick Grune (dick) at dick
For long text sequences (say hundreds of thousands of tokens),
the hashing is not really efficient any more since too many
spurious matches occur. Therefore, the forward reference table is
scanned a second time, eliminating from any chain all references to
runs that do not end in the same token. For the UNIX manuals this
reduced the number of matches from 91.9% to 1.9% (of which 0.06%
were genuine).
30-Aug-1988 Dick Grune (dick) at dick
For compatibility, NextTop has been rewritten to yield true or
false and to accept a pointer to a run as a parameter.
30-Aug-1988 Dick Grune (dick) at dick
When trying to find line-number and lseek position to beginnings
and ends of runs found, the whole set of runs was scanned for each
line in each file. Now only the runs belonging to that file are
scanned; to this end another linked list has been braided through
the data structures (tx_chunk).
30-Aug-1988 Dick Grune (dick) at dick
The longest-common-substring algorithm was called much too often,
mainly because the forward references made by hashing suffered from
pollution. If you have say 1000 tokens and a hash range of say
10000, about 5 % of the hashings will be false matches, i.e. 50
matches, which is quite a lot on a natural number of 2 to 3 matches.
Improved by doing a second check in make_forw_ref().
12-Jun-1988 Dick Grune (dick) at dick
Installed a Lisp version supplied by Gertjan Akkerman.
15-Jan-1988 Dick Grune (dick) at dick
Added register declarations all over the place.
14-Jan-1988 Dick Grune (dick) at dick
It is often useful to match a piece of code exactly, especially
when function names (or, even more so, macro names) are involved.
What one would want is having all the letters in the text array,
but this is kind of hard, since each entry is one lexical item.
This means that under the -F option each letter is a lex item, and
normally each tag is a lex item; this requires two lex grammars in
one program; no good. So, on the -F flag we hash the identifier
into one lex item, which is hopefully characteristic enough. It
works.
30-Sep-1987 Dick Grune (dick) at dick
Some cosmetics.
31-Aug-1987 Dick Grune (dick) at dick
Moved the whole thing to the SUN (while testing on a VAX and a
MC68000)
16-Aug-1987 Dick Grune (dick) at dick
The test program lang.c is no longer a main program, but rather a
subroutine called in main() in sim.c, through the command line
option -1 or -2.
23-Apr-1987 Dick Grune (dick) at tjalk
Changed the name 'index' into 'elem_of', because of compatibility
problems on different Unices. Added a declaration for it in
the file algollike.c
10-Mar-1987 Dick Grune (dick) at tjalk
Changed the printing of the header of a run so that:
- long file names will no longer be truncated
- the run length is displayed
27-Jan-1987 Dick Grune (dick) at tjalk
Switched it right off again! Getting them in textual order is
still more unpleasant, since now you cannot find the important
ones if their are more than a few runs.
27-Jan-1987 Dick Grune (dick) at tjalk
Going to experiment with leaving out the sorting; just all the
runs, in the order we meet them. Should be as good or better.
Comparisons of more than 100 runs are very rare anyway, so the
fact that those over a 100 are rejected is probably no great
help. Getting them in a funny order is a nuisance, however. Down
with featurism. Just to be safe, present version saved as
870127.SV
26-Dec-1986 Dick Grune (dick) at tjalk
Names of overall parameters in params.h changed to more uniformity.
26-Dec-1986 Dick Grune (dick) at tjalk
Since the top package and the instantiation system have grown
apart so much, I have integrated the old top package into sim,
i.e., done the instantiation by hand. This removes top.g and
top.p, and will save outsiders from wondering what is going on
here.
23-Dec-1986 Dick Grune (dick) at tjalk
Use setbuf to print unbuffered while reading the files (lex core
dumps, other mishaps) and print buffered while printing the real
output (for speed).
30-Nov-1986 Dick Grune (dick) at tjalk
Various small changes in *lang.l:
; ignored conditionally (!options['f'])
new format for tokens in struct idf
cosmetics: macro Layout, macro UnsafeComChar, no \n
in character denotations, more than one char
in a char denotations in Pascal, etc.
30-Nov-1986 Dick Grune (dick) at tjalk
Added a Modula-2 version.
29-Nov-1986 Dick Grune (dick) at tjalk
Restricting tokens to the ASCII95 character set is really too
severe: some languages have many more reserved words (COBOL!).
Corrected this by adding a couple of '&0377' in strategic places.
Added a routine for printing the 8-bit beasties: show_token().
15-Aug-1986 Dick Grune (dick) at tjalk
Since the ; is superfluous in both C and Pascal, it is now ignored
by clang.l and pascallang.l
15-Aug-1986 Dick Grune (dick) at tjalk
The code in CheckRun in Xlang.l was incorrect in that it used the
wrong criterion for throwing away trailing garbage. I've taken
CheckRun etc. out of the Xlang.l-s and turned them into a module
"algollike.c". Made a cleaner interface and avoided duplication of
code.
02-Jul-1986 Dick Grune (dick) at tjalk
Looking backwards in compare.c to see if we are in the middle of a
run is an atavism. You can be and still be all right, e.g., if
part of the run was rejected as not fitting for a function.
Removed from compare.c.
10-Jun-1986 Dick Grune (dick) at tjalk
The function hash_code() in hash.c could yield a negative value;
corrected.
09-Jun-1986 Dick Grune (dick) at tjalk
Changed the name of the file text.h to sim.h. Sim.h is more
appropriate and text.h sounds as if it belongs to text.l, with
which it has no connection.
04-Jun-1986 Dick Grune (dick) at tjalk
After having looked at a couple of hash functions and having done
some calculations on the number of duplicates normally encountered
in hash functions, I conclude that our function in hash.c is quite
good. Removed all the statistics-gathering stuff.
Actually, hash_table[] is not the hash table at all; it is a
forward reference table; likewise, the real hash table was called
last[]. Renamed both.
There is a way to keep the hash table local without putting it on
the stack: use malloc().
02-Jun-1986 Dick Grune (dick) at tjalk
Added a simple lex file for text: each word is condensed into a
hash code which is mapped on the ASCII95 character set. This
turns out to be quite effective.
01-Jun-1986 Dick Grune (dick) at tjalk
The macros cput(tk) and c_eol() both have a return in them, so any
code after them may not be executed -> they have to be last in an
entry. But they weren't, in many places; I can't imagine why it
all worked nevertheless. They have been renamed return_tk(tk) and
return_eol() and the entries have been restructured.
30-May-1986 Dick Grune (dick) at tjalk
Moved the string and character entries in clang.l and pascallang.l
to a place behind the comment entries, to avoid strings (and
characters) being recognized inside comments. I first thought
this would not happen, but as Maarten pointed out, if both
interpretations have the same length, lex will take the first
entry. Now this will happen if the string occupies the whole line
that would otherwise be taken as a comment. In short,
/*
"hallo"
*/
would return ".
28-May-1986 Dick Grune (dick) at tjalk
Added -d option, to display the output in diff(1) format (courtesy
of Maarten van der Meulen).
Rewrote the lexical parsing of comments (likewise courtesy Maarten
van der Meulen).
20-May-1986 Dick Grune (dick) at tjalk
Added a routine to convert identifiers to lower case in
pascallang.l .
19-May-1986 Dick Grune (dick) at tjalk
Added -a option, to quickly check antecedent of a file (courtesy
of Maarten van der Meulen).
18-May-1986 Dick Grune (dick) at tjalk
Brought everything under RCS/CVS.
18-Mar-1986 Dick Grune (dick) at tjalk
Added modifications by Paul Bame (hp-lsd!paul@hp-labs) to have an
option -w to set the page width.
21-Feb-1986 Dick Grune (dick) at tjalk
Took array last[N_HASH] out of make_hash() in hash.c, due to stack
overflow on the Gould (reported by George Walker
tekig4!georgew@mcvax.uucp)
16-Feb-1986 Dick Grune (dick) at tjalk
Corrected some subtractions that caused unsigned ints to turn
pseudo-negative. (Reported by jaap@mcvax)
11-Jan-1986 Dick Grune (dick) at tjalk
Touched up for distribution.
10-Jan-1986 Dick Grune (dick) at tjalk
Fill_line was not called for empty lines, which caused them to be
printed as repetitions of the previous line.
24-Dec-1985 Dick Grune (dick) at tjalk
Reduced hash table to a single array of indices; it is used only
in one place, which makes it very easy to make it (the hash table)
optional. General tune-up of everything. This seems to be
another stable "final" version.
14-Dec-1985 Dick Grune (dick) at tjalk
Some experiments with hash formulas:
h = (h OP CST) + *p++ OP CST yields right wrong
* 96 - 32 205 562
* 96 - 2 205 560
* 96 205 560
* 97 205 559
<< 0 66 3128
<< 1 203 555
<< 2 205 536
<< 7 203 540
Conclusion: it doesn't matter, unless you do it wrong.
01-Oct-1983 Dick Grune (dick) at vu44
Oldest known files.
# This file is part of the software similarity tester SIM.
# Written by Dick Grune, Vrije Universiteit, Amsterdam.
# $Id: ChangeLog,v 2.33 2017-03-19 09:30:37 dick Exp $
#

166
core/sim/sim_3_01/ForEachFile.c Normal file
View File

@@ -0,0 +1,166 @@
/* This file is part of the auxiliaries library.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: ForEachFile.c,v 1.23 2016-02-22 08:20:42 Gebruiker Exp $
*/
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <errno.h>
#include "ForEachFile.h"
/* Library module source prelude */
#undef _FOREACHFILE_CODE_
#ifndef lint
#define _FOREACHFILE_CODE_
#endif
#ifdef LIB
#define _FOREACHFILE_CODE_
#endif
#ifdef _FOREACHFILE_CODE_
/* Library module source code */
/* TREE SCANNING */
#ifdef S_IFLNK /* system with symbolic links */
#define LSTAT lstat
#else /* S_IFLNK */
#define LSTAT Stat
#endif /* S_IFLNK */
int
is_dirstat(const struct stat *fs) {
if (!fs) return 0;
return ((fs->st_mode & S_IFMT) == S_IFDIR);
}
int
is_Dirname(const Fchar *Fn) {
if (!Fn) return 0;
struct stat stb;
if (LSTAT(Fn, &stb) < 0) return 0;
return ((stb.st_mode & S_IFMT) == S_IFDIR);
}
int
is_Admin_Dirname(const Fchar *Fn) {
if (!Fn) return 0;
return Fnamecmp(Fn, str2Fname(".")) == 0
|| Fnamecmp(Fn, str2Fname("..")) == 0;
}
static void do_dir(
Fchar *Fn,
int (*proc)(const Fchar *, const char *, const struct stat *)
);
static void
do_name(Fchar *Fn,
int (*proc)(const Fchar *, const char *, const struct stat *),
int top_level
) {
/* examine Fn */
struct stat fs;
if (LSTAT(Fn, &fs) < 0) {
(void)(*proc)(Fn, strerror(errno), 0);
return;
}
/* report on Fn and get possible return code */
int rc = (*proc)(Fn, (char*)0, &fs);
if (!is_dirstat(&fs)) return;
/* Fn is a directory, so rc may be meaningful */
if (!top_level) if (!rc) return;
#ifdef S_IFLNK
/* don't follow links */
if ((fs.st_mode & S_IFMT) == S_IFLNK) return;
#endif
do_dir(Fn, proc);
}
static void
do_dir(
Fchar *Fn,
int (*proc)(const Fchar *, const char *, const struct stat *)
) {
/* treat directory */
Dir_t *dir = Opendir(Fn);
if (dir == 0) {
(void)(*proc)(Fn, "directory not readable", 0);
return;
}
/* scan new directory */
/* append separator */
int Fn_len = Fnamelen(Fn);
Fn[Fn_len++] = '/';
Fn[Fn_len] = '\0';
/* descend */
Dirent_t *dent;
while ((dent = Readdir(dir)) != (Dirent_t *)0) {
const Fchar *d_name = dent->d_name;
if (is_Admin_Dirname(d_name)) continue;
/* append name */
Fnamecat(Fn, d_name);
do_name(Fn, proc, 0);
/* remove appended name*/
Fn[Fn_len] = '\0';
}
/* remove appended separator*/
Fn[--Fn_len] = '\0';
Closedir(dir);
}
static MSDOS_sep = (Fchar)'\\';
static UNIX_sep = (Fchar)'/';
static void
clean_name(Fchar *Fn) {
/* remove a trailing separator */
int Fn_len = Fnamelen(Fn);
if (Fn_len > 1 && (Fn[Fn_len-1] == MSDOS_sep || Fn[Fn_len-1] == UNIX_sep)) {
Fn[Fn_len-1] = '\0';
}
}
/* THE ENTRIES */
void
ForEachFile(
const Fchar *Fname,
int (*proc)(const Fchar *, const char *, const struct stat *)
) {
if (!Fname || !Fname[0] || !proc) return; /* just to make sure */
/* get Fn */
Fchar Fn[MAX_FILE_NAME_LENGTH];
Fnamecpy(Fn, Fname);
clean_name(Fn);
/* top level */
do_name(Fn, proc, 1);
}
/* End library module source code */
#endif /* _FOREACHFILE_CODE_ */
#ifdef lint
static void
satisfy_lint(void *x) {
(void)is_dirstat(0);
(void)is_Dirname(0);
(void)is_Admin_Dirname(0);
ForEachFile(0, 0);
satisfy_lint(x);
}
#endif /* lint */

52
core/sim/sim_3_01/ForEachFile.h Normal file
View File

@@ -0,0 +1,52 @@
/* This file is part of the auxiliaries library.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: ForEachFile.h,v 1.14 2017-01-22 14:49:59 Gebruiker Exp $
*/
#ifndef _FOREACHFILE_H_
#define _FOREACHFILE_H_
#include "fname.h"
#include <sys/types.h>
#include <sys/stat.h>
/****
* ForEachFile(const Fchar *Fn, int (*proc)(...):
Each file or directory reachable from Fn is passed to the procedure proc(),
which is declared as:
int proc(const Fchar *Fn, const char *msg, const struct stat *fs):
the file or directory Fn has been reached;
if msg != NULL, an error prevails the text of which is *msg;
otherwise fs points to the stat buffer for Fn.
If Fn is a file or the argument of ForEachFile(), the return value of
proc() is ignored; if it is a directory but not the argument of
ForEachFile() and the return value is 0, the directory is not visited
further, and files and directories in it are not reported.
Basically if proc() always returns 1, all reachable files are reported;
if it always returns 0, only the local files and directories are
reported. But finer control is possible.
* proc() is not called with the directory names "." or ".." unless it is the
first argument to ForEachFile().
* MAX_FILE_NAME_LENGTH is the maximum length of the file name Fn, including
directories.
****/
/* Public entries */
#define MAX_FILE_NAME_LENGTH 1024 /* maximum file name length */
extern void ForEachFile(
const Fchar *Fname,
int (*proc)(const Fchar *, const char *, const struct stat *)
);
/* avoid awkward dir test */
extern int is_dirstat(const struct stat *fs);
extern int is_Dirname(const Fchar *Fn);
extern int is_Admin_Dirname(const Fchar *Fn); /* !=0 for "." and ".." */
#endif /* _FOREACHFILE_H_ */

17
core/sim/sim_3_01/Korean1.txt Normal file
View File

@@ -0,0 +1,17 @@
Lee
노인은 신체적, 정신적. 경제적으로 의존성이 높은
취약계층이며 만성 퇴행성 질환의 유병률이 높고, 장
기적인 치료 및 요양보호와 포괄적인 보건의료 서비
스의 수요가 높은 집단이다(Sunwoo, 2008).
2009년 통계청에 따르면 우리나라 전체 인구 가운
데 65세 이상 인구는 10.7%로 고령화 사회이며, 건강
심사 평가원의 2010년 상반기 진료비 통계지표에서
노인 인구의 진료비는 2009년 상반기 대비 15.5% 증
가 했고, 전체 진료비의 31.4%를 차지하여, 노인 의
료비의 비중이 매우 높은 것을 알 수 있다.(Health
Insurance Review & Assessment Service, 2010) ©≠𒌩우리
나라에서 노인 부양은 핵가족화와 여성의 사회 활동
참여가 증가하고 가정에서 노인을 돌볼 수 없어 장기
요양 기관에서 간호와 관리를 하게 되는 경향이 증가

15
core/sim/sim_3_01/Korean2.txt Normal file
View File

@@ -0,0 +1,15 @@
노인 인구의 진료비는 2009년 상반기 대비 15.5% 증
가 했고, 전체 진료비의 31.4%를 차지하여, 노인 의
료비의 비중이 매우 높은 것을 알 수 있다.(Health
Insurance Review & Assessment Service, 2010) ©≠𒌩우리
나라에서 노인 부양은 핵가족화와 여성의 사회 활동
참여가 증가하고 가정에서 노인을 돌볼 수 없어 장기
요양 기관에서 간호와 관리를 하게 되는 경향이 증가
노인은 신체적, 정신적. 경제적으로 의존성이 높은
취약계층이며 만성 퇴행성 질환의 유병률이 높고, 장
기적인 치료 및 요양보호와 포괄적인 보건의료 서비
스의 수요가 높은 집단이다(Sunwoo, 2008).
2009년 통계청에 따르면 우리나라 전체 인구 가운
데 65세 이상 인구는 10.7%로 고령화 사회이며, 건강
심사 평가원의 2010년 상반기 진료비 통계지표에서

31
core/sim/sim_3_01/LICENSE.txt Normal file
View File

@@ -0,0 +1,31 @@
Copyright (c) 1986, 2007, Dick Grune, Vrije Universiteit, The Netherlands
All rights reserved.
Redistribution and use in source and binary forms,
with or without modification, are permitted provided
that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
* Neither the name of the Vrije Universiteit nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

544
core/sim/sim_3_01/Makefile Normal file
View File

@@ -0,0 +1,544 @@
# This file is part of the software similarity tester SIM.
# Written by Dick Grune, Vrije Universiteit, Amsterdam.
# $Id: Makefile,v 2.97 2017-03-19 09:49:29 dick Exp $
#
VERSION="-DVERSION=\"3.0.1 of 2017-08-15\""
# E N T R Y P O I N T S
help:
@echo 'Entry points:'
@echo 'test: compile sim_c and run a simple test'
@echo ''
@echo 'binaries: create all binaries'
@echo 'exes: create executables in MSDOS'
@echo 'install: install all binaries'
@echo ''
@echo 'view_man: view sim.pdf'
@echo 'lint: lint sim sources'
@echo 'simsim: run sim_c on the sim sources'
@echo 'view_SPC: view the percentage computation document'
@echo 'chklat: do a LaTeX check on the .tex documents'
@echo ''
@echo 'fresh: remove created files'
#
# When you modify any of the following macros, do 'make clean'
#
# System dependencies
# =============== including ../lib/sysidf.mk here
# This file is part of the auxiliary libraries.
# Written by Dick Grune, dick@dickgrune.com
# $Id: sysidf.mk,v 1.18 2016-04-22 15:11:58 dick Exp $
#
################################################################
# For UNIX-like systems
SYSTEM = UNIX
SUBSYSTEM = SOLARIS
# Locations
DIR = /usr
BINDIR = $(DIR)/bin.`$(DIR)/bin/arch`
MAN1DIR = $(DIR)/man/man1
# Commands
COPY = cp -p
EXE = .exe#
LEX = flex
LN = ln
ZIP = zip -o
###############################################################
# General, C compilation:
CC = gcc -D$(SYSTEM) -D$(SUBSYSTEM)
LINT = lint -ansi -D$(SYSTEM) -D$(SUBSYSTEM)
LINTFLAGS = -xh
# General, text:
LATEX = pdflatex
SHOW_PDF = evince
SHOW_PDF = acroread
SHOW_PDF = pdfview
GROFF = groff -man
GROFF = man2pdf
.SUFFIXES: .1 .3 .pdf
.1.pdf:
$(GROFF) $<
.3.pdf:
$(GROFF) $<
# =============== end of ../lib/sysidf.mk
# Compiling
MEMORY = -DMEMCHECK -DMEMCLOBBER
CFLAGS = $(VERSION) $(MEMORY) -O4
LIBFLAGS = #
LINTFLAGS = $(MEMORY) -h# -X
LOADFLAGS = -s# # strip symbol table
LOADER = $(CC) $(LOADFLAGS)
# Debugging
CFLAGS += -DDEBUG
DEBUG_C = debug.c
DEBUG_O = debug.o
DEBUG_H = debug.h
# T E S T P A R A M E T E R S
# Rumen Stevanov test
TEST_LANG = text
TEST_OPT = -pPae -r4 -O -t4
TEST_INP = Contributors/Rumen_Stefanov/new/*.txt
# slash test
TEST_LANG = text
TEST_OPT = -r24 -M clang.c pascallang.c "|" textlang.c
TEST_INP =
# spaced word test
TEST_LANG = text
TEST_OPT = -r 5
TEST_INP = testfiles/test_seplet
# -i option test
TEST_LANG = c
TEST_OPT = -f -r 20 -R -i <option-i.inp
TEST_INP = #
# overlap test, foo^100 vs. foo^150
TEST_LANG = text
TEST_OPT = -r50 -p
TEST_OPT = -r50
TEST_INP = foo_100
TEST_INP = foo_100 foo_150
# single file test
TEST_LANG = c
TEST_OPT = -r24 -M
TEST_INP = pass3.c pass3.c # compares 1st to 2nd, then 2nd to 2nd
# tight match test, foo_100 has exactly 100 tokens
TEST_LANG = text
TEST_OPT = -r50
TEST_INP = foo_100
# percentage test
TEST_LANG = c
TEST_OPT = -puae
TEST_INP = *.l
# larger test
TEST_LANG = text
TEST_OPT = -r24 -M -puae
TEST_INP = clang.c textlang.c
TEST_INP = textlang.c clang.c javalang.c
# test UTF-8 text
TEST_LANG = text
TEST_OPT = -r12 -MO -T -w80 -d
TEST_INP = Korean1.txt Korean2.txt
# test UTF-8 program
TEST_LANG = c
TEST_OPT = -r12 -MO -w80 -d
TEST_INP = utf8test.c utf8test.c
# C++ test
TEST_LANG = c++
TEST_OPT = -r8 -MO -
TEST_INP = c++test.c++ c++test.c++# # for lack of real C++ files
# test
TEST_LANG = c
TEST_OPT = -r10 -MO -
TEST_INP = clang.c pascallang.c
TEST_INP = pass2.c pass1.c
# I N T R O D U C T I O N
# Each module (set of programs that together perform some function)
# has the following sets of files defined for it:
# _SRC the source files, from which other files derive
# _CFS the C-files, from which the object files derive
# _OBJ object files
# _GRB garbage files produced by the module
#
# (This is a feeble attempt at software-engineering a Makefile.)
#
test: sim.res stream.res percentages.res version.res
# B I N A R I E S
BINARIES = sim_c$(EXE) sim_c++$(EXE) sim_java$(EXE) sim_pasc$(EXE) \
sim_m2$(EXE) sim_lisp$(EXE) sim_mira$(EXE) sim_text$(EXE)
binaries: $(BINARIES)
EXES = sim_c.exe sim_c++.exe sim_java.exe sim_pasc.exe \
sim_m2.exe sim_lisp.exe sim_mira.exe sim_text.exe
exes: $(EXES)
# A U X I L I A R Y M O D U L E S
# Common modules:
COM_CFS = token.c lex.c stream.c text.c tokenarray.c $(DEBUG_C) \
ForEachFile.c fname.c Malloc.c any_int.c
COM_OBJ = token.o lex.o stream.o text.o tokenarray.o $(DEBUG_O) \
ForEachFile.o fname.o Malloc.o any_int.o
COM_SRC = token.h lex.h stream.h text.h tokenarray.h $(DEBUG_H) \
ForEachFile.h fname.h Malloc.h any_int.h \
lang.h language.h \
sortlist.spc sortlist.bdy system.par $(COM_CFS)
# C files for the abstract modules:
ABS_CFS = lang.c language.c
# The idf module:
IDF_CFS = idf.c
IDF_OBJ = idf.o
IDF_SRC = idf.h $(IDF_CFS)
# The runs package:
RUNS_CFS = runs.c percentages.c
RUNS_OBJ = runs.o percentages.o
RUNS_SRC = runs.h percentages.h $(RUNS_CFS)
# The main program:
MAIN_CFS = sim.c options.c newargs.c hash.c compare.c add_run.c \
pass1.c pass2.c pass3.c
MAIN_OBJ = sim.o options.o newargs.o hash.o compare.o add_run.o \
pass1.o pass2.o pass3.o
MAIN_SRC = sim.h options.h newargs.h hash.h compare.h add_run.h \
pass1.h pass2.h pass3.h \
debug.par settings.par $(MAIN_CFS)
sim.o: Makefile # because of $(VERSION)
# The similarity tester without the language part:
SIM_CFS = $(COM_CFS) $(IDF_CFS) $(RUNS_CFS) $(MAIN_CFS)
SIM_OBJ = $(COM_OBJ) $(IDF_OBJ) $(RUNS_OBJ) $(MAIN_OBJ)
SIM_SRC = $(COM_SRC) $(IDF_SRC) $(RUNS_SRC) $(MAIN_SRC)
# L A N G U A G E S
# The algollike module:
ALG_CFS = algollike.c
ALG_OBJ = algollike.o
ALG_SRC = algollike.h $(ALG_CFS)
# The C Language module: # C
CLANG_CFS = clang.c
CLANG_OBJ = clang.o
CLANG_SRC = clang.l
clang.c: clang.l
$(LEX) -t clang.l >$@
SIM_C_CFS = $(SIM_CFS) $(ALG_CFS) $(CLANG_CFS)
SIM_C_OBJ = $(SIM_OBJ) $(ALG_OBJ) $(CLANG_OBJ)
sim_c$(EXE): $(SIM_C_OBJ)
$(LOADER) $(SIM_C_OBJ) -o $@
SIM_GRB += clang.c sim_c
$(BINDIR)/sim_c$(EXE): sim_c$(EXE)
$(COPY) sim_c$(EXE) $@
# The C++ Language module: # C++
C++LANG_CFS = c++lang.c
C++LANG_OBJ = c++lang.o
C++LANG_SRC = c++lang.l
c++lang.c: c++lang.l
$(LEX) -t c++lang.l >$@
SIM_C++_CFS = $(SIM_CFS) $(ALG_CFS) $(C++LANG_CFS)
SIM_C++_OBJ = $(SIM_OBJ) $(ALG_OBJ) $(C++LANG_OBJ)
sim_c++$(EXE): $(SIM_C++_OBJ)
$(LOADER) $(SIM_C++_OBJ) -o $@
SIM_GRB += c++lang.c sim_c++
$(BINDIR)/sim_c++$(EXE): sim_c++$(EXE)
$(COPY) sim_c++$(EXE) $@
# The Java Language module: # Java
JAVALANG_CFS = javalang.c
JAVALANG_OBJ = javalang.o
JAVALANG_SRC = javalang.l
javalang.c: javalang.l
$(LEX) -t javalang.l >$@
SIM_JAVA_CFS = $(SIM_CFS) $(ALG_CFS) $(JAVALANG_CFS)
SIM_JAVA_OBJ = $(SIM_OBJ) $(ALG_OBJ) $(JAVALANG_OBJ)
sim_java$(EXE): $(SIM_JAVA_OBJ)
$(LOADER) $(SIM_JAVA_OBJ) -o $@
SIM_GRB += javalang.c sim_java
$(BINDIR)/sim_java$(EXE): sim_java$(EXE)
$(COPY) sim_java$(EXE) $@
# The Pascal Language module: # Pascal
PASCLANG_CFS = pascallang.c
PASCLANG_OBJ = pascallang.o
PASCLANG_SRC = pascallang.l
pascallang.c: pascallang.l
$(LEX) -t pascallang.l >pascallang.c
SIM_PASC_CFS = $(SIM_CFS) $(ALG_CFS) $(PASCLANG_CFS)
SIM_PASC_OBJ = $(SIM_OBJ) $(ALG_OBJ) $(PASCLANG_OBJ)
sim_pasc$(EXE): $(SIM_PASC_OBJ)
$(LOADER) $(SIM_PASC_OBJ) -o $@
SIM_GRB += pascallang.c sim_pasc
$(BINDIR)/sim_pasc$(EXE): sim_pasc$(EXE)
$(COPY) sim_pasc$(EXE) $@
# The Modula-2 Language module: # Modula-2
M2LANG_CFS = m2lang.c
M2LANG_OBJ = m2lang.o
M2LANG_SRC = m2lang.l
m2lang.c: m2lang.l
$(LEX) -t m2lang.l >$@
SIM_M2_CFS = $(SIM_CFS) $(ALG_CFS) $(M2LANG_CFS)
SIM_M2_OBJ = $(SIM_OBJ) $(ALG_OBJ) $(M2LANG_OBJ)
sim_m2$(EXE): $(SIM_M2_OBJ)
$(LOADER) $(SIM_M2_OBJ) -o $@
SIM_GRB += m2lang.c sim_m2
$(BINDIR)/sim_m2$(EXE): sim_m2$(EXE)
$(COPY) sim_m2$(EXE) $@
# The Lisp Language module: # Lisp
LISPLANG_CFS = lisplang.c
LISPLANG_OBJ = lisplang.o
LISPLANG_SRC = lisplang.l
lisplang.c: lisplang.l
$(LEX) -t lisplang.l >$@
SIM_LISP_CFS = $(SIM_CFS) $(ALG_CFS) $(LISPLANG_CFS)
SIM_LISP_OBJ = $(SIM_OBJ) $(ALG_OBJ) $(LISPLANG_OBJ)
sim_lisp$(EXE): $(SIM_LISP_OBJ)
$(LOADER) $(SIM_LISP_OBJ) -o $@
SIM_GRB += lisplang.c sim_lisp
$(BINDIR)/sim_lisp$(EXE): sim_lisp$(EXE)
$(COPY) sim_lisp$(EXE) $@
# The Miranda Language module: # Miranda
MIRALANG_CFS = miralang.c
MIRALANG_OBJ = miralang.o
MIRALANG_SRC = miralang.l
miralang.c: miralang.l
$(LEX) -t miralang.l >$@
SIM_MIRA_CFS = $(SIM_CFS) $(ALG_CFS) $(MIRALANG_CFS)
SIM_MIRA_OBJ = $(SIM_OBJ) $(ALG_OBJ) $(MIRALANG_OBJ)
sim_mira$(EXE): $(SIM_MIRA_OBJ)
$(LOADER) $(SIM_MIRA_OBJ) -o $@
SIM_GRB += miralang.c sim_mira
$(BINDIR)/sim_mira$(EXE): sim_mira$(EXE)
$(COPY) sim_mira$(EXE) $@
# The Text module: # Text
TEXTLANG_CFS = textlang.c
TEXTLANG_OBJ = textlang.o
TEXTLANG_SRC = textlang.l
textlang.c: textlang.l
$(LEX) -t textlang.l >$@
SIM_TEXT_CFS = $(SIM_CFS) $(TEXTLANG_CFS)
SIM_TEXT_OBJ = $(SIM_OBJ) $(TEXTLANG_OBJ)
sim_text$(EXE): $(SIM_TEXT_OBJ)
$(LOADER) $(SIM_TEXT_OBJ) -o $@
SIM_GRB += textlang.c sim_text
$(BINDIR)/sim_text$(EXE): sim_text$(EXE)
$(COPY) sim_text$(EXE) $@
# T E S T S
# Some simple tests:
.PHONY: sim.res percentages.res
sim.res: sim_$(TEST_LANG)$(EXE) $(TEST_INP)
./sim_$(TEST_LANG)$(EXE) $(TEST_OPT) $(TEST_INP)
stream.res: sim_$(TEST_LANG)$(EXE)
./sim_$(TEST_LANG)$(EXE) -- $(TEST_OPT) $(TEST_INP) >$@
wc $@ $(TEST_INP)
TEST_GRB = stream.res
PERC_TEST_EXE = sim_text$(EXE)
PERC_TEST_FILES = foo_100 foo_150
PERC_TEST_EXE = sim_c$(EXE)
PERC_TEST_FILES = pascallang.l clang.l javalang.l
percentages.res:$(PERC_TEST_EXE) $(PERC_TEST_FILES)
@echo ''
./$(PERC_TEST_EXE) -T -p $(PERC_TEST_FILES)
@echo ''
./$(PERC_TEST_EXE) -T -pa $(PERC_TEST_FILES)
@echo ''
./$(PERC_TEST_EXE) -T -pae $(PERC_TEST_FILES)
version.res: sim_$(TEST_LANG)$(EXE)
./sim_$(TEST_LANG)$(EXE) -v
# More simple tests, using the C version only:
simsim: sim_c$(EXE) $(SIM_CFS) $(ALG_CFS)
./sim_c$(EXE) -fr 20 $(SIM_CFS) $(ALG_CFS)
# Lint
lint: $(SIM_SRC) $(ALG_SRC) $(ABS_CFS)
$(LINT) $(LINTFLAGS) $(SIM_CFS) $(ALG_CFS) $(ABS_CFS)
# O T H E R E N T R I E S
# Documentation
view_man: sim.pdf
$(SHOW_PDF) sim.pdf
%.pdf: %.tex
$(LATEX) $< || log2errmsg <$*.log
$(LATEX) $<
$(LATEX) $<
SPC = Similarity_Percentage_Computation
view_SPC: $(SPC).pdf
$(SHOW_PDF) $(SPC).pdf
DOCS += $(SPC).pdf
# LaTeX checker
chklat:
chklat *.tex
# Installation
install_all: install # just a synonym
install: $(MAN1DIR)/sim.1 \
$(BINDIR)/sim_c$(EXE) \
$(BINDIR)/sim_c++$(EXE) \
$(BINDIR)/sim_java$(EXE) \
$(BINDIR)/sim_pasc$(EXE) \
$(BINDIR)/sim_m2$(EXE) \
$(BINDIR)/sim_lisp$(EXE) \
$(BINDIR)/sim_mira$(EXE) \
$(BINDIR)/sim_text$(EXE)
$(MAN1DIR)/sim.1: sim.1
$(COPY) sim.1 $@
# Clean-up
.PHONY: clean fresh
clean:
-rm -f *.o
-rm -f $(SIM_GRB)
-rm -f $(TEST_GRB)
-rm -f *.aux *.log *.out
-rm -f a.out a.exe sim.txt core mon.out
fresh: clean
-rm -f *.exe
# D E P E N D E N C I E S
# DO NOT DELETE THIS LINE -- make depend depends on it.
ForEachFile.o: ForEachFile.c ForEachFile.h fname.h
Malloc.o: Malloc.c any_int.h Malloc.h
add_run.o: add_run.c sim.h text.h runs.h percentages.h options.h \
add_run.h
algollike.o: algollike.c sim.h options.h token.h algollike.h
any_int.o: any_int.c any_int.h
c++lang.o: c++lang.c options.h token.h language.h algollike.h idf.h lex.h \
lang.h
clang.o: clang.c options.h token.h language.h algollike.h idf.h lex.h \
lang.h
compare.o: compare.c sim.h text.h token.h tokenarray.h hash.h language.h \
options.h add_run.h compare.h debug.par
count_sim_dup.o: count_sim_dup.c
debug.o: debug.c debug.h
fname.o: fname.c fname.h
hash.o: hash.c system.par debug.par sim.h text.h Malloc.h any_int.h \
token.h language.h tokenarray.h options.h hash.h
idf.o: idf.c system.par token.h idf.h
javalang.o: javalang.c options.h token.h language.h algollike.h idf.h \
lex.h lang.h
lang.o: lang.c token.h language.h algollike.h idf.h lex.h lang.h
language.o: language.c token.h language.h
lex.o: lex.c lex.h
lisplang.o: lisplang.c token.h language.h algollike.h lex.h lang.h idf.h
m.o: m.c
m2lang.o: m2lang.c options.h token.h language.h algollike.h idf.h lex.h \
lang.h
miralang.o: miralang.c token.h language.h algollike.h lex.h lang.h idf.h
newargs.o: newargs.c sim.h ForEachFile.h fname.h Malloc.h newargs.h
options.o: options.c sim.h token.h language.h options.h
pascallang.o: pascallang.c options.h token.h language.h algollike.h idf.h \
lex.h lang.h
pass1.o: pass1.c debug.par sim.h text.h token.h tokenarray.h lang.h \
options.h pass1.h
pass2.o: pass2.c debug.par sim.h token.h text.h lang.h pass2.h \
sortlist.bdy
pass3.o: pass3.c system.par settings.par debug.par sim.h text.h token.h \
runs.h options.h pass3.h percentages.h
percentages.o: percentages.c debug.par sim.h text.h options.h Malloc.h \
percentages.h sortlist.bdy
runs.o: runs.c sim.h text.h runs.h Malloc.h debug.par sortlist.bdy
sim.o: sim.c system.par settings.par sim.h options.h newargs.h token.h \
tokenarray.h language.h text.h runs.h hash.h compare.h pass1.h pass2.h \
pass3.h percentages.h stream.h lang.h Malloc.h any_int.h
stream.o: stream.c system.par sim.h options.h token.h lang.h stream.h
t.o: t.c
text.o: text.c debug.par sim.h token.h stream.h lang.h Malloc.h options.h \
text.h
textlang.o: textlang.c sim.h token.h idf.h lex.h lang.h language.h
token.o: token.c token.h
tokenarray.o: tokenarray.c sim.h Malloc.h token.h lang.h tokenarray.h
utf8test.o: utf8test.c

417
core/sim/sim_3_01/Malloc.c Normal file
View File

@@ -0,0 +1,417 @@
/* This file is part of the checked memory manager MALLOC.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: Malloc.c,v 1.24 2017-01-24 19:46:57 Gebruiker Exp $
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include "any_int.h"
#include "Malloc.h"
/* make malloc.h available */
#undef malloc
#undef calloc
#undef realloc
#undef free
/*Library module source prelude */
#undef _MALLOC_CODE_
#ifndef lint
#define _MALLOC_CODE_
#endif
#ifdef LIB
#define _MALLOC_CODE_
#endif
#ifdef _MALLOC_CODE_
/* Library module source code */
#undef new
#define new use_my_new /* don't call Malloc in Malloc.c */
#define my_new(type) ((type *)malloc(sizeof (type)))
/* All output goes through designated files, so we block printf, etc. */
#undef printf
#define printf use_fprintf
#undef putchar
#define putchar use_fprintf
static size_t restricted_balance = 0; /* to simulate out-of-memory */
static void
fprintloc(FILE *out, const char *fname, int l_nmb) {
fprintf(out, "\"%s\", line %d: ", fname, l_nmb);
}
void
_out_of_memory(const char *msg, const char *fname, int l_nmb, size_t size) {
fprintloc(stderr, fname, l_nmb);
fprintf(stderr, "OUT OF MEMORY");
if (msg) {
fprintf(stderr, ": %s", msg);
}
if (size != 0) {
fprintf(stderr, ", requested size = %s bytes",
any_uint2string(size, 0));
}
fprintf(stderr, "\n");
fflush(stderr);
ReportMemoryStatus(stderr);
exit(1);
}
/* ADMINISTRATION */
static vlong_uint total = 0;
static vlong_uint balance = 0;
static vlong_uint max = 0;
struct alloc { /* corresponds to an allocated block */
struct alloc *next;
const char *addr;
size_t size;
const char *fname;
int l_nmb;
};
#define HASH_SIZE 16381 /* largest prime under 2^16 */
static struct alloc *alloc_bucket[HASH_SIZE];
#define alloc_bucket_for(x) alloc_bucket[((unsigned int)(x)%HASH_SIZE)]
static void
register_alloc(char *addr, size_t size, const char *fname, int l_nmb) {
/* registers the allocation of a block in the administration */
struct alloc *new;
struct alloc **al_hook = &alloc_bucket_for(addr);
if (addr == 0) return;
new = my_new(struct alloc);
new->addr = addr;
new->size = size;
new->fname = fname; /* no need to copy fname */
new->l_nmb = l_nmb;
new->next = *al_hook;
*al_hook = new;
total += size;
balance += size;
if (balance > max) {
max = balance;
}
}
static struct alloc **
pointer_to_alloc_for(const char *addr) {
struct alloc **al_hook = &alloc_bucket_for(addr);
while (*al_hook) {
if ((*al_hook)->addr == addr) break;
al_hook = &(*al_hook)->next;
}
return al_hook;
}
static size_t
register_free(char *addr) {
/* registers the freeing of a block */
struct alloc **old_p = pointer_to_alloc_for(addr);
struct alloc *old = *old_p;
if (old == 0) return (size_t) -1;
size_t old_size = old->size;
*old_p = old->next;
free((void *)old);
balance -= old_size;
return old_size;
}
void
MemClobber(void *p, size_t size) {
unsigned char *s = (unsigned char *)p;
size_t i;
for (i = 0; i < size; i++) {
s[i] = 0125; /* 0101 0101 */
}
}
/* MEMORY STATUS */
struct call { /* summarizes all the allocations at a call in the program */
struct call *next;
const char *fname;
int l_nmb;
unsigned int n_blocks;
int var_size; /* all blocks have the same size or not */
size_t size; /* !var_size: the one size; var_size: sum of sizes */
};
static struct call *
compacted_calls(void) {
struct call *list_of_calls = 0;
int i;
for (i = 0; i < HASH_SIZE; i++) {
struct alloc *al = alloc_bucket[i];
while (al) {
struct call *cl = list_of_calls;
/* try to find a call entry for this program location */
while (cl) {
if ( cl->fname == al->fname
&& cl->l_nmb == al->l_nmb
) break;
cl = cl->next;
}
if (cl) {
/* this is known call; update */
if (cl->var_size) {
cl->size += al->size;
}
else if (cl->size != al->size) {
/* switch to var_size */
cl->var_size = 1;
cl->size =
cl->n_blocks*cl->size + al->size;
}
cl->n_blocks++;
}
else { /* this is a new call */
cl = my_new(struct call);
cl->fname = al->fname;
cl->l_nmb = al->l_nmb;
cl->n_blocks = 1;
cl->var_size = 0;
cl->size = al->size;
/* prepend to list_of_calls */
cl->next = list_of_calls;
list_of_calls = cl;
}
al = al->next;
}
}
return list_of_calls;
}
static int
number_of_calls(const struct call *cl) {
int res = 0;
while (cl != 0) {
res++;
cl = cl->next;
}
return res;
}
static void
report_actual_call(FILE *out, const struct call *cl) {
fprintloc(out, cl->fname, cl->l_nmb);
fprintf(out, "still allocated: %d block%s of size ",
cl->n_blocks, (cl->n_blocks == 1 ? "" : "s")
);
if (cl->var_size) {
/* cl->size is the sum of the sizes */
size_t av = (cl->size+cl->n_blocks/2) / cl->n_blocks;
fprintf(out, "%s on average", any_uint2string(av, 0));
if (cl->n_blocks > 1) {
fprintf(out, " = %s", any_uint2string(cl->size, 0));
}
}
else {
/* cl->size is the single size */
fprintf(out, "%s", any_uint2string(cl->size, 0));
if (cl->n_blocks > 1) {
vlong_uint all = cl->size*cl->n_blocks;
fprintf(out, " = %s", any_uint2string(all, 0));
}
}
fprintf(out, "\n");
}
static void
report_actual_calls(FILE *out) {
const struct call *cl = compacted_calls(); /* allocates cl */
int n_calls = number_of_calls(cl);
if (n_calls == 0) return;
fprintf(out, "There %s %d call position%s with unreclaimed memory:\n",
(n_calls == 1 ? "is" : "are"),
n_calls,
(n_calls == 1 ? "" : "s")
);
while (cl) {
report_actual_call(out, cl);
struct call *next_cl = cl->next;
free((void *)cl); /* frees cl */
cl = next_cl;
}
}
void
ReportMemoryStatus(FILE *out) {
if (out == 0) out = stderr;
report_actual_calls(out);
fprintf(out, "Total memory allocated = %s", any_uint2string(total, 0));
fprintf(out, ", max. allocated = %s", any_uint2string(max, 0));
fprintf(out, ", still allocated = %s", any_uint2string(balance, 0));
fprintf(out, "\n");
fflush(out);
}
/* MALLOC */
void *
_mreg_malloc(int chk, size_t size, const char *fname, int l_nmb) {
void *res;
if (restricted_balance && balance + size > restricted_balance) {
res = 0;
} else {
res = malloc(size);
}
if (res == 0) {
if (chk) {
_out_of_memory(0, fname, l_nmb, size);
/*NOTREACHED*/
}
return res;
}
register_alloc(res, size, fname, l_nmb);
#ifdef MEMCLOBBER
MemClobber((char *)res, size);
#endif /* MEMCLOBBER */
return res;
}
void *
_mreg_calloc(int chk, size_t n, size_t size, const char *fname, int l_nmb) {
void *res;
if (restricted_balance && balance + n*size > restricted_balance) {
res = 0;
} else {
res = calloc(n, size);
}
if (res == 0) {
if (chk) {
_out_of_memory(0, fname, l_nmb, n*size);
/*NOTREACHED*/
}
return res;
}
register_alloc(res, n*size, fname, l_nmb);
return res;
}
void *
_mreg_realloc(int chk, void *addr, size_t size, const char *fname, int l_nmb) {
void *res;
size_t old_size = register_free(addr);
/* we report first, because the realloc() below may cause a crash */
if ( /* we are not reallocating address 0, which is allowed */
addr != 0
&& /* the address was never handed out before */
old_size == (size_t) -1
) {
fprintloc(stderr, fname, l_nmb);
fprintf(stderr, ">>>> unallocated block reallocated <<<<\n");
fflush(stderr);
}
if (restricted_balance && balance + size > restricted_balance) {
res = 0;
} else {
res = realloc(addr, size);
}
if (res == 0) {
if (chk) {
_out_of_memory(0, fname, l_nmb, size);
/*NOTREACHED*/
}
return res;
}
register_alloc(res, size, fname, l_nmb);
#ifdef MEMCLOBBER
if (old_size > 0 && size > old_size) {
MemClobber(((char *)res)+old_size, size-old_size);
}
#endif /* MEMCLOBBER */
return res;
}
/* ARGSUSED */
void
_mreg_free(void *addr, const char *fname, int l_nmb) {
size_t old_size = register_free(addr);
/* we report first, because the free() below may cause a crash */
if (old_size == (size_t) -1) {
fprintloc(stderr, fname, l_nmb);
fprintf(stderr, ">>>> unallocated block freed ");
fprintf(stderr, "or multiple free of allocated block <<<<\n");
fflush(stderr);
}
else {
#ifdef MEMCLOBBER
MemClobber((char *)addr, old_size);
#endif /* MEMCLOBBER */
}
free(addr);
}
char *
_new_string(int chk, const char *s, const char *fname, int l_nmb) {
return strcpy((char *)(_mreg_malloc(chk, strlen(s)+1, fname, l_nmb)),
s);
}
/* End library module source code */
#endif /* _MALLOC_CODE_ */
#ifdef lint
static void
satisfy_lint(void *x) {
void *v;
v = _mreg_malloc(0, 0, 0, 0);
v = _mreg_calloc(0, 0, 0, 0, 0);
v = _mreg_realloc(0, 0, 0, 0, 0);
_mreg_free(x, 0, 0);
OutOfMemoryExit(0);
ReportMemoryStatus(0);
MemClobber(v, 0);
v = _new_string(0, 0, 0, 0);
satisfy_lint(v);
}
#endif /* lint */

120
core/sim/sim_3_01/Malloc.h Normal file
View File

@@ -0,0 +1,120 @@
/* This file is part of the checked memory manager MALLOC.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: Malloc.h,v 1.14 2017-01-23 13:30:23 Gebruiker Exp $
*/
#ifndef _MALLOC_H_
#define _MALLOC_H_
/*****
The files Malloc.[ch] provide several functionalities:
- checking for "out of memory": to simplify programming
- allocating memory using a routine new(type): " " " "
- clobbering freshly allocated memory: to obtain safer programs
- checking for freeing of unallocated blocks: " " " "
- reporting on memory usage: to obtain cleaner programs
- detecting memory leaks: " " " "
The module defines several sets of routines:
1. void *Malloc(size_t s)
void *Calloc(size_t n, size_t s)
void *Realloc(void *p, size_t s)
void Free(void *p)
These routines act like checking versions of their Unix counterparts,
except that they never return NULL; upon out-of-memory an error message
is given on standard error, showing the file name and the line number
of the call.
Since in almost all cases there is nothing more intelligent to do, this
is almost always adequate, and makes for simpler and safer programming.
2. void *TryMalloc(size_t s)
void *TryCalloc(size_t n, size_t s)
void *TryRealloc(void *p, size_t s)
void OutOfMemoryExit(const char *msg)
In those rare cases that the program *can* continue when out of memory, the
Try... routines can be used; they act exactly like those in group 1,
except that they return NULL when out of memory.
A call of OutOfMemoryExit(msg) gives an out-of-memory error message,
displaying the message msg and some memory usage information, and
terminating the program.
3. T *new(T)
T *try_new(T)
char *new_string(const char *s)
A call of new(T), with T any type, yields a pointer of type T* to a block
of type T, allocated using Malloc(). A call of try_new(T) does the same
but returns NULL when out of memory.
A call of new_string(s), with s a string, yields a pointer to a copy of s,
allocated using Malloc(); it is equivalent to strdup() except that it uses
Malloc(). A call of try_new_string(s) does the same but returns NULL when
out of memory.
4. void ReportMemoryStatus(FILE *f)
Produces a compacted list of allocated but not yet freed blocks on the
stream f, with information about where they were allocated.
This is useful to get insight into memory use and abuse.
5. void MemClobber(void *p, size_t size)
When Malloc.c is compiled with -DMEMCLOBBER, it clobbers all newly allocated
memory from Malloc() and Realloc() just after allocation, and all freed
memory just before freeing it. An area is clobbered by overwriting it with
a wacky bit pattern. This is done in the hope that improper use of memory
will cause some evident error somewhere.
The routine that performs the clobbering, MemClobber(void *p, size_t size),
is available regardless of the -DMEMCLOBBER compilation option. It can be
used to create comparison patterns.
Notes:
* Compiled with any of the -DMEM... flags, Malloc will also produce run-time
error messages for multiple Free()s of the same block, and Realloc()s on
not-allocated blocks. It then allows the program to continue.
* The system consumes hardly any time and is fast enough to be kept active
at all times.
*****/
/* Blocking malloc.h */
#define malloc(s) you_are_using_the_Malloc_package_so_use_Malloc
#define calloc(n,s) you_are_using_the_Malloc_package_so_use_Calloc
#define realloc(p,s) you_are_using_the_Malloc_package_so_use_Realloc
#define free(p) you_are_using_the_Malloc_package_so_use_Free
/* Private entries */
extern void *_mreg_malloc(int chk, size_t size, const char *fname, int l_nmb);
extern void *_mreg_calloc(int chk, size_t n, size_t size, const char *fname, int l_nmb);
extern void *_mreg_realloc(int chk, void *addr, size_t size, const char *fname, int l_nmb);
extern void _mreg_free(void *addr, const char *fname, int l_nmb);
extern void _out_of_memory(
const char *msg, const char *fname, int l_nmb, size_t size);
extern char *_new_string(int chk, const char *s, const char *fname, int l_nmb);
/* Public entries */
#define Malloc(s) (_mreg_malloc(1, (s), __FILE__, __LINE__))
#define Calloc(n,s) (_mreg_calloc(1, (n), (s), __FILE__, __LINE__))
#define Realloc(p,s) (_mreg_realloc(1, (void *)(p), (s), __FILE__, __LINE__))
#define Free(p) (_mreg_free((void *)(p), __FILE__, __LINE__))
#define TryMalloc(s) (_mreg_malloc(0, (s), __FILE__, __LINE__))
#define TryCalloc(n,s) (_mreg_calloc(0, (n), (s), __FILE__, __LINE__))
#define TryRealloc(p,s) (_mreg_realloc(0, (void *)(p), (s), __FILE__, __LINE__))
#define OutOfMemoryExit(s) _out_of_memory((s), __FILE__, __LINE__, 0)
#define new(type) ((type *)Malloc(sizeof (type)))
#define try_new(type) ((type *)TryMalloc(sizeof (type)))
#define new_string(s) (_new_string(1, (s), __FILE__, __LINE__))
#define try_new_string(s)(_new_string(0, (s), __FILE__, __LINE__))
extern void ReportMemoryStatus(FILE *f);
extern void MemClobber(void *p, size_t size);
#endif /* _MALLOC_H_ */

62
core/sim/sim_3_01/README Normal file
View File

@@ -0,0 +1,62 @@
# This file is part of the software similarity tester SIM.
# Written by Dick Grune, Vrije Universiteit, Amsterdam.
# $Id: README,v 2.18 2017-03-19 09:42:55 dick Exp $
These programs test for similar or equal stretches in one or more program
or text files and can be used to detect common code or plagiarism. See sim.pdf.
Checkers are available for C, C++, Java, Pascal, Modula-2, Lisp, Miranda and
natural language text.
>>>> NEW, March 2017:
- C++ testing added
>>>> NEW, May 2016:
- better percentage computation
>>>> NEW, Jan 2014:
- 64-bit compatible
- works also on 32-bit machines with software 64-bit emulator
- accepts | as new-old separator
==== To install on any system with gcc, flex, cp, ln, echo, rm, and wc,
or their equivalents, for example UNIX/Linux or MSDOS+MinGW:
Unpack the archive sim_3_*.zip
To compile and test, edit the Makefile to fit the local situation, and call:
make test
This will generate one executable called sim_c, the checker for C, and will
run two small tests to show sample output.
To install, examine the Makefile, edit BINDIR and MAN1DIR to sensible paths,
and call
make install
To change defaults, adjust the file settings.par and recompile.
==== To install on MSDOS, if you don't have a C compiler, the archive
sim_exe_3_*.zip contains:
SIM_C.EXE similarity tester for C
SIM_C++.EXE similarity tester for C++
SIM_JAVA.EXE similarity tester for Java
SIM_PASC.EXE similarity tester for Pascal
SIM_M2.EXE similarity tester for Modula-2
SIM_LISP.EXE similarity tester for Lisp
SIM_MIRA.EXE similarity tester for Miranda
SIM_TEXT.EXE similarity tester for text
==== To extend:
To add another language L, write a file Llang.l along the lines of clang.l
or the other *lang.l files, extend the Makefile and recompile.
All knowledge about a given language L is located in Llang.l; the rest of
the program expects each token to be a 16-bit character.
Dick Grune
email: dick@dickgrune.com
http://www.dickgrune.com

477
core/sim/sim_3_01/Similarity_Percentage_Computation.tex Normal file
View File

@@ -0,0 +1,477 @@
% This file is part of the software similarity tester SIM.
% Written by Dick Grune, dick@dickgrune.com
% $Id: Similarity_Percentage_Computation.tex,v 2.5 2016-05-27 08:04:29 dick Exp $
%
\batchmode
\documentclass[a4paper,fleqn]{article}
\newcommand{\mytitle}{Similarity Percentage Computation in SIM}
\newcommand{\myshorttitle}{SPC}
\title{\mytitle}
\author{Dick Grune \\
dick@dickgrune.com}
\date{May 18, 2016}
\renewcommand{\sim}[0]{{\it sim}}
\newcommand{\Sim}[0]{{\it Sim}}
\newcommand{\bd}[1]{\textbf{#1}}
\newcommand{\cw}[1]{\texttt{#1}}
\newcommand{\Bs}[0]{\ensuremath{\backslash}}
\newcommand{\OK}{\ding{52}}
\newcommand{\KO}{\ding{56}}
\newcommand{\ignore}[1]{}
\begin{document}
\maketitle
\section{Introduction}
The similarity testing facility of {\sim} has been working satisfactorily
since its inception in 1986, but the computation of similarity percentages,
introduced in 2001, has been a continuous source of headache.
There seem to be two reasons for this. The first is that there is no clear
definition of the notion ``similarity percentage''.
The second is that all
attempts to obtain similarity percentages have resulted in nonsensical results
(often percentages far exceeding 100\%), or have required quadratic time, or
sometimes both. Quadratic time requirements are unacceptable because they
make {\sim} unsuitable for large-scale text comparisons like
plagiarism detection, one of {\sim}'s largest application areas.
\section{The Definition of ``Similarity Percentage''}
Normally {\sim} supplies the information
\begin{quote}
Files $F_1$ and $F_2$ have matches
$(T_{1,1}, T_{1,2})$, $(T_{2,1}, T_{2,2}), ..., (T_{k,1}, T_{k,2})$
\end{quote}
where the text segments $T$-s in the matches have the form $(F, p, l)$,
designating the text in file $F$, starting at position $p$ with length $l$,
if we view a file as an array of tokens.
The matches have the following properties (assuming a minimum run length of 1):
\begin{itemize}
\item
For any match $((F_1, p_1, l_1), (F_2, p_2, l_2))$ we have $l_1 = l_2$ and
$F_1[p_1..p_1+l_1] = F_2[p_2..p_2+l_2]$ (i.e. a match matches equal segments).
\item
For each token $F_1[a]$ for which there is a token $F_2[b]$ such that
$F_1[a]=F_2[b]$, there is a match $((F_1, p_1, l_1), (F_2, p_2, l_2))$ such
that $p_1 \leq a < p_1+l_1$ and $p_2 \leq b < p_2+l_2$
(i.e. if two files have a token in common, it is present in a match).
\item
The segments $T_{1,1}$, $T_{2,1}, ..., T_{k,1}$ do not overlap. (This does not
apply to $T_{1,2}$, $T_{2,2}, ..., T_{k,2}$.)
\item
If there is a match $((F_1, p_1, l_1), (F_2, p_2, l_2))$ there are no matches
$((F_1, p_1 + l_1, ...), (F_2, ..., ...))$ or
$((F_1, p_1-l, l), (F_2, ..., ...))$ (i.e. no match could be extended on either
end, i.e. each match is maximal in length).
\end{itemize}
This information is well-defined and unambiguous.
It is produced by {\sim} in almost linear time.
Unfortunately this information does not reasonably extend to something like
\begin{quote}
Files $F_1$ and $F_2$ have $N$\% material in common.
\end{quote}
One is tempted to conclude that $F_1$ and $F_2$ have material of length
$l_1 + l_2$ +... + $l_k$ in common, but $l_1 + l_2$ +... + $l_k$ can easily be
larger than $F_1$ or $F_2$, resulting in similarities of over 100\%.
This can, f.e., happen when $F_1=BB$ and $F_2=B$, where $B$ is a block of text
with length $l_B$.
Then the matches found between the two files are
$((F_1, 0, l_B), (F_2, 0, l_B))$ and $((F_1, l_B, l_B), (F_2, 0, l_B))$.
So the two files have a common length of $l_B+l_B = 2l_B$, which gives $F_2$
(with length $l_B$) a similarity percentage of 200\%.
A more fundamental approach to similarity is needed.
The fullest and clearest representation of what an $N$-token file $F_1$ and an
$M$-token file $F_2$ have in common is an $N{\times}M$ Boolean correlation
matrix $C$ with $C[n,m]=True$ if $F_1[n]=F_2[m]$ and $C[n,m]=False$ elsewhere.
Suppose $F_1$=\cw{"a b c d e f g h i j"} (length = 10) and
$F_2$=\cw{"f g h v w x y z a b c d e"} (length = 13).
Then the correlation matrix $C$ is
\newlength{\pluswidth}
\settowidth{\pluswidth}{$~+$}
\begin{tabular}{lc*{13}{p{\pluswidth}@{}}}
& & & & & & & \multicolumn{2}{c}{$F_2$} \\
& &f&g&h&v&w&x&y&z&a&b&c&d&e\\
&a& & & & & & & & &+& & & & \\
&b& & & & & & & & & &+& & & \\
&c& & & & & & & & & & &+& & \\
&d& & & & & & & & & & & &+& \\
$F_1$ &e& & & & & & & & & & & & &+\\
&f&+& & & & & & & & & & & & \\
&g& &+& & & & & & & & & & & \\
&h& & &+& & & & & & & & & & \\
&i& & & & & & & & & & & & & \\
&j& & & & & & & & & & & & & \\
\end{tabular}
If we project the pluses onto the top and left axes, we get an indication of
how much material $F_1$ can be found in $F_2$ and vice versa:
\begin{tabular}{lcc*{13}{p{\pluswidth}@{}}}
& & & & & & & & \multicolumn{2}{c}{$F_2$} \\
& & &f&g&h&v&w&x&y&z&a&b&c&d&e\\
& & &*&*&*& & & & & &*&*&*&*&*\\
&a&*& & & & & & & & &+& & & & \\
&b&*& & & & & & & & & &+& & & \\
&c&*& & & & & & & & & & &+& & \\
&d&*& & & & & & & & & & & &+& \\
$F_1$ &e&*& & & & & & & & & & & & &+\\
&f&*&+& & & & & & & & & & & & \\
&g&*& &+& & & & & & & & & & & \\
&h&*& & &+& & & & & & & & & & \\
&i& & & & & & & & & & & & & & \\
&j& & & & & & & & & & & & & & \\
\end{tabular}
\noindent
From this representation we see that
\begin{quote}
$F_1$ consists for 8/10 = 80.0\% of $F_2$ material \\
$F_2$ consists for 8/13 = 61.5\% of $F_1$ material \\
\end{quote}
It is this information that {\sim} tries -- and frequently fails -- to
provide.
\section{The Comparison Machine}
To see how this information can be obtained by {\sim}
or why it cannot, we have to have a closer look at {\sim}'s comparison machine.
{\Sim} cannot use the $N{\times}M$ matrix. Its size is quadratic in the
number of tokens in the combined files to be compared, which would restrict
the total number of tokens to 100.000 at the utmost. Instead it uses a linear
array of tokens, with some additional data, as follows.
\subsection{The Algorithm}
The comparison machine (the routine \cw{lcs()} in \cw{compare.c}) uses two
indexes into $T$: \cw{i0} and \cw{i1}.
The index \cw{i0} steps through the array, and for each position the
index \cw{i1} is used to hunt for the match
$((F_1, i_0, l), (F_2, i_1, l))$ with largest $l$ and $i_0 < i_1$.
The skeleton algorithm is
\small%
\begin{verbatim}
i0 := 0;
WHILE i0 < end_of_text
i1 := 0;
WHILE i1 < end_of_text
IF T[i0] = T[i1]
find longest match starting from(i0, i1)
IF longer than previous longest match starting from(i0)
store as longest match starting from(i0)
i1 := i1 + 1
IF there is a longest match starting from(i0)
i0 := i0 + length of longest match starting from(i0)
ELSE
i0 := i0 + 1
\end{verbatim}%
\normalsize%
\noindent
In short, the inner loop finds long matches; the outer loop finds the
longest of these long matches.
Suppose we have two files $F_1 = $\cw{"abcdefabc"} and
$F_2 = $\cw{"fabcdef"}; the algorithm then identifies the matches
\begin{quote}
$((F_1,0,6), (F_2,1,6))$ (matching \cw{"abcdef"}), \\
$((F_1,6,3), (F_2,1,3))$ (matching \cw{"abc"}).
\end{quote}
If the files had been offered in the reverse order, the matches would have
been
\begin{quote}
$((F_2,0,4), (F_1,5,4))$ (matching \cw{"fabc"}), \\
$((F_2,4,3), (F_1,3,3))$ (matching \cw{"def"}),
\end{quote}
which shows that the results of the comparison algorithm are sensitive to the
order in which the files are presented.
\subsection{Details of the Increments}
Even after finding a match in the inner loop, \cw{i1} should only be
incremented by 1, since right after a good match a better match may be found.
Suppose $F_1$=\cw{"aaabcd"} and $F_2$=\cw{"aaaabcd"}.
With \cw{i0=0} and \cw{i1=7} (just at the beginning of $F_2$) we find the
match
\begin{quote}
$((F_1,0,3), (F_2,0,3))$ (matching \cw{"aaa"}),
\end{quote}
but with \cw{i1=8} we find
\begin{quote}
$((F_1,0,6), (F_2,1,6))$ (matching \cw{"aaabcd"}),
\end{quote}
which is a better match starting at \cw{i0=0}.
When a longest match is found in the outer loop, \cw{i0} must be increased by
the length of that match, to avoid repetitive matches.
Suppose we have one file $F$=\cw{"aaaaaaaa"}.
Then the first match will be
\begin{quote}
$((F,0,4), (F,4,4))$ (matching \cw{"aaaa"}).
\end{quote}
When we increase \cw{i0} in steps of 1, the rest of the matches will be
\begin{quote}
$((F,1,3), (F,4,3))$ (matching \cw{"aaa"}), \\
$((F,2,3), (F,5,3))$ (matching \cw{"aaa"}), \\
$((F,3,2), (F,5,2))$ (matching \cw{"aa"}), \\
$((F,4,2), (F,6,2))$ (matching \cw{"aa"}), \\
$((F,5,1), (F,6,1))$ (matching \cw{"a"}), \\
$((F,6,1), (F,7,1))$ (matching \cw{"a"}),
\end{quote}
for a total of $O(N)$ matches, many of them repeating
earlier information.
If, however, we increase \cw{i0} by the length of the match found, the rest of
the matches are
\begin{quote}
$((F,4,2), (F,6,2))$ (matching \cw{"aa"}), \\
$((F,6,1), (F,7,1))$ (matching \cw{"a"}),
\end{quote}
for a total of $O(\log N)$ matches, all informative.
The actual algorithm has many other features: establishing a minimum match
length, options for avoiding to compare a file to itself, etc.
There is one important optimization, concerned with incrementing \cw{i1}.
Incrementing \cw{i1} by 1 makes the algorithm quadratic in the number of
tokens, which is unacceptable.
We have seen, however, that sometimes increasing by 1 is necessary.
The problem is heuristically solved by having
an array \cw{Forward\_Reference[]}, which for each position in the token array
gives the index of the nearest larger position in the token array where
matching text of at least the minimum required length can be found.
This array is constructed in linear time by a prescan,
using hashing extensively.
Rather than incrementing \cw{i1} by 1 when looking for the next match, \cw{i1}
is set to \cw{Forward\_Reference[i1]}.
In principle using this forward references array does not take away the
quadratic component, but it multiplies it by $\phi$, the density of the
similarities.
And since we usually have $\phi \ll 1$, this optimization makes {\sim}
usable.
The linear component of course remains.
\section{Problems and their Causes}
The arrangement described above is not directly suitable for percentage
computation, for three reasons.
\begin{enumerate}
\item
The contents of $F_k$ are explained in terms of matches with files
$F_{k+1}, ..., F_n$; matches from $F_1, ..., F_{k-1}$ are not noticed.
\item
Not all matches in $F_{k+1}, ..., F_n$ are noticed.
\item
Only a single match for a block at a given position in $F_k$ is reported;
other lesser matches or equal matches further on are not noticed.
\end{enumerate}
Each of these problems will now be considered in turn.
\subsection{Ignoring Earlier Files}
Earlier files are ignored because the \cw{Forward\_Reference} array is exactly
that, it references forward. Normally this is not a problem, since any
similarities between $F_k$ and an earlier $F_p$ (with
$ 1 \leq p \leq {k-1}$) will be have been noted when $F_p$ was analyzed.
One may be tempted to try to deduce the contribution of a file $F_p$
($ 1 \leq p \leq {k-1}$) to $F_k$ from matches of the form
$((F_p,,),(F_k,,))$,
but that is not possible.
Suppose $F_p=BaBBc$ and $F_k=aBc$, where $B$ is a block of text of size $l_B$.
The algorithm provides us with the matches
\begin{quote}
$((F_p,0,l_B),(F_k,1,l_B))$ (matching $B$),\\
$((F_p,l_B,1+l_B),(F_k,0,1+l_B))$ (matching $aB$),\\
$((F_p,l_B+1+l_B,l_B+1),(F_k,1,l_B+1))$ (matching $Bc$)
\end{quote}
Note that in none of the matches the text in $F_p$ overlaps.
These matches are perfect for determining the $N$ in the statement
``$F_p$ consists for $N$\% of $F_k$ material'':
$N = (l_B+1+l_B+l_B+1) / (l_B+1+l_B+l_B+1) = (3l_B+2) / (3l_B+2) = 100\%$.
But if we rely on these matches to compute the $N$ in the statement
``$F_k$ consists for $N$\% of $F_p$ material'', we arrive at the conclusion
that $N = (l_B+1+l_B+l_B+1) / (l_B+2) = (3l_B+2) / (l_B+2) \approx 300\%$.
It is clear that this is caused by the overlap of the matches in $F_k$, but
this example makes it equally clear that it would be very difficult to
disentangle such overlaps in the general case.
\subsection{Selectively Ignoring Later Files}
When there are a number of files, say four, each containing somewhere an
identical block of text $B$, say $F_1=abBcd,F_2=efgB,F_3=Bhi,F_4=jklBmno$
(where $a..o$ are letters not occurring in $B$)
{\sim} produces the following three matches
\begin{quote}
$((F_1,2,l_B),(F_2,3,l_B))$, \\
$((F_2,3,l_B),(F_3,0,l_B))$, \\
$((F_3,0,l_B),(F_4,3,l_B))$
\end{quote}
producing a linear number of matches, which is very desirable for most
applications.
But it means that the matches
\begin{quote}
$((F_1,2,l_B),(F_3,0,l_B))$, \\
$((F_1,2,l_B),(F_4,3,l_B))$, \\
$((F_2,3,l_B),(F_4,3,l_B))$
\end{quote}
are missed.
So percentages between $F_1$ and $F_2$, $F_2$ and $F_3$, and $F_3$ and $F_4$
could possibly be derived from these matches, but those between
$F_1$ and $F_2$, $F_1$ and $F_4$, and $F_2$ and $F_4$ cannot.
As we have noted before, the output of {\sim} is sensitive to the order in
which the files are presented.
\subsection{Ignoring Lesser Matches}
Given the three files $F_1=abBcd,F_2=efgB,F_3=bBhi$,
{\sim} produces the following two matches
\begin{quote}
$((F_1,1,l_B+1),(F_3,0,l_B+1))$, (matching $bB$) \\
$((F_2,3,l_B),(F_3,1,l_B))$, (matching $B$)
\end{quote}
but the lesser match $((F_1,2,l_B),(F_2,3,l_B))$, relating $F_1$ to $F_2$, is
missed.
This is acceptable in similarity testing since the $B$ in $F_2$ is caught by
$((F_2,3,l_B),(F_3,1,l_B))$, but for similarity percentage computation we need
all matches.
\section{Solutions}
\subsection{Accessing Earlier Files}
There are at least two ways to obtain both matches of the form
$((F_p,,),(F_q,,))$ \emph{and} of the form $((F_q,,),(F_p,,))$ with $p < q$:
modifying the order in which the files are processed, and modifying the
forward reference system.
\subsubsection{Modifying the File Order}
Running the entire program again with the order of the files reversed will
provide the missing $((F_q,,),(F_p,,))$, at the expense of doubling the
running time.
\subsubsection{Modifying the Forward Referencing System}
Each forward reference chain is particular to a specific string $S$ of text,
with $|S|$ the minimum required match length.
The chain starts from the first occurrence of $S$ in the text array, then
leads to one or more subsequent occurrences of $S$, and ends in a NULL pointer
at the last occurrence of $S$ in the text array.
When at a position in $F_p$ we start using the forward reference chain for say
$S$, we can reach only positions in $F_q$ with $q > p$ (not $ q \ge p$,
since when computing percentages a file is not compared to itself).
Now suppose the chain, rather than terminating with a NULL pointer at the last
occurrence of $S$, looped back to its beginning.
Then when working on $F_p$ we do not stop at the last occurrence of $S$ in the
text array while hunting for $S$
but continue at the first occurrence of $S$ in the text array, from where we
may access files $F_q$ with $ q < p $.
This statistically doubles the length of the chain, doubling the work of the
comparison machine; but the effort in the preparation of the text array is not
doubled, as it would be if the program were run twice.
A circular list is an awkward data structure and requires careful programming.
During construction of a forward reference chain it can easily be made to loop,
by remembering where it started and then rather than terminating it with a NULL
pointer end it with a pointer to the starting point.
When using the chain, following it must stop when we reach the position from
which we started (which is certainly on the chain).
So the problem of how to access earlier files can be solved at the expense of
roughly doubling the run time.
\subsection{Full Coverage of Later Files}
The problem is caused by the original algorithm hunting for the largest match
in the rest of the files, whereas for percentage computation it should look
for any match in each of the other files separately.
The possibility of comparing each file with each other file separately has
already been provided by the \bd{-e} option.
Again this results in a quadratic time requirement, and again we invoke the
low density of actual matches to reduce the weight of the problem.
The original algorithm produces the matches in a left-to-right largest match
order, to minimize the number of matches; for the percentage computation we
just want them all.
Since the nature and number of matches depends on the order in which they
are identified, one might wonder if the left-to-right largest match order may
cause problems to percentage computation.
Suppose $F_p$=\cw{"abcdxe"} and $F_q$=\cw{"abcdxxe"}, with a required
minimum run length of 2.
The matches found are
\begin{verbatim}
abcdxe abcdxxe
abcdx abcdx
e e
\end{verbatim}
with the result that $F_q$ consists for $5/7 = 71.4\%$ out of $F_p$ material.
The match \cw{e} is ignored because it does not have the required minimum
length.
If the files were compared in the reverse order the following matches would be
obtained
\begin{verbatim}
abcdxxe abcdxe
abcdx abcdx
xe xe
\end{verbatim}
and $F_q$ would consist for $(5+2)/7 = 100\%$ of $F_p$ material
Considerations:
\begin{itemize}
\item
The example is contrived and hinges on there being a minimum required length
and on multiply overlapping patterns.
No examples seem to exist without these ingredients.
\item
Multiply overlapping patterns do occur in practice, f.e. in lists of
identifiers or numbers, but these are usually not very informative and of
little interest.
\item
The algorithm for finding the maximum similarity percentage may conceivably
require exhaustive search and be exponential.
\end{itemize}
It seems reasonable to ignore this problem for the moment, and use the normal
{\sim} matching algorithm in comparing one file to one file\footnote{%
The problem did show up in test runs, manifesting itself by minor differences
in the percentages depending on the order of the input files, but went away
when the run length was set to 1.}.
\subsection{Lesser Matches}
Since lesser matches can only occur when three or more files are involved,
using the \cw{-e} option solves this problem too, since it restricts all
comparisons to two files only.
\section{Conclusion}
Using circular forward reference chains and the \cw{-e}-option, correct
similarity percentages can be obtained; the algorithm has a moderate quadratic
component though.
A compromise can be obtained by omitting the \cw{-e} option, which is the
greatest source of quadratic behaviour, and use circular forward reference
chains only. This makes the algorithm almost linear again, but will
underreport percentages.
\end{document}

214
core/sim/sim_3_01/TechnReport Normal file
View File

@@ -0,0 +1,214 @@
CONCISE REPORT ON THE ALGORITHMS IN SIM 970623
INTRODUCTION
The general outline of the similarity checker is as follows:
1. the files are read in (pass 1)
2. a forward-reference table is prepared
3. the set of interesting runs is determined
4. the line numbers of the runs are determined (pass 2)
5. the contents of the runs are printed in order (pass 3)
To keep the memory requirements (relatively) small, the exact positions
of the tokens are not recorded. This necessitates pass 2. See, however,
the pertinent chapter.
READING THE FILES
Each file is tokenized using an lex-generated scanner appropriate for
the input. Each token fits in one byte, possibly using all 8 bits. The
tokens are stored in the array Token_Array[], which is extended by
reallocation if it overflows. See tokenarray.c.
Also, to optimize away pass 2, an attempt is made to remember the token
positions of all beginnings of lines. The token-positions at BOL are
stored in the array nl_buff[], which is also extended by reallocation,
if needed. If the attempt fails due to lack of memory, nl_buff[] is
abandoned, and pass2 will read the files instead.
PREPARING THE FORWARD-REFERENCE TABLE
Text is compared by comparing every substring to all substrings
to the right of it; this process is in essence quadratic. However,
only substrings of length at least 'MinRunSize' are of interest,
which gives us the possibility to speed up this process by using
a hash table.
Once the entire text has been read in, a forward-reference table
forward_references[] is made (see hash.c).
For every position in the text, we construct an index which gives
the next position in the text where a run of MinRunSize tokens
starts that has the same hash code. If there is no such run, the
index is 0.
To fill in this array, we use a hash table last_index[], such that
last_index[i] is the index of the latest token with hash_code i, or 0 if
there is none. If at a given position p, we find that the text ahead of
us has hash code i, last_index[i] tells us which position in
forward_references[] will have to be updated to p.
See MakeForward_References().
For long text sequences (say hundreds of thousands of tokens), the
hashing is not really efficient any more since too many spurious matches
occur. Therefore, the forward reference table is scanned a second time,
eliminating from any chain all references to runs that do not start with
and end in the same token (actually this is a second hash code).
For the UNIX manuals this reduced the number of matches from 91.9% to 1.9%
(of which 0.06% was genuine).
DETERMINING THE SET OF INTERESTING RUNS
The overall structure of the routine Compare_Files() (see compare.c) is:
for all new files
for all texts it must be compared to
for all positions in the new file
for all positions in the text
for ever increasing sizes
try to match and keep the best
If for a given position in the new file a good run (i.e. on of at least
minimum length) has been found, the run is registered using a call of
add_run(), the run is skipped in the new file and searching continues at
the position after it. This prevents duplicate reports of runs.
Add_run() allocates a struct run for the run (see sim.h)
which contains two struct chunks and a quality description. It fills
in the two chunks with the pertinent info, one for the first file and
one for the second (which may be the same, if the run relates two chunks
in the same file).
The run is then entered into the arbitrary-in-sorted-out store AISO (see
aiso.spc and aiso.bdy, a genuine generic abstract data type in C!), in
which it is inserted according to its quality. Both positions
(struct position) in both chunks in the run (so four in total) are each
entered in a linked list starting at the tx_pos field in the struct text
of the appropriate file.
When this is finished, the forward reference table can be deleted.
So the final results of this phase are visible both through the tx_pos
fields and through the aiso interface.
DETERMINING THE EXACT POSITION OF EACH RUN (PASS 2)
The purpose of this pass is to find for each chunk, which up to now is
known by token position only, its starting and ending line number (which
cannot be easily derived from the token position).
For each file that has a non-zero tx_pos field, ie. that has some
interesting chunks, the positions in the tx_pos list are sorted on
ascending line number (they have been found in essentially arbitrary
order) by sort_pos() in pass2.c.
Next we scan the pos list and the file in parallel, updating the info in
a position when we meet it. A position carries an indication whether it
is a starting or an ending position, since slightly differing
calculations have to be done in each case.
Actually, if the nl_buff[] data structure still exists, the file is not
accessed at all and the data from nl_buff[] is used instead. This is
done transparently in buff.c.
PRINTING THE CONTENTS OF THE RUNS (PASS 3)
Since each struct run has now been completely filled in, this is simple;
the hard work is calculating the page layout.
Pass3() accesses the aiso store and retrieves from it the runs in
descending order of importance. Show_run() opens both files, positions
them using the line numbers and prints the runs.
================================================================
CODE EXCERPT OF THE SOFTWARE SIMILARITY TESTER SIM (980222)
sim:
get command line options
check the options
init language, to precompute tables
pass1, read the files
# there is an array Token_Array[] that holds all input tokens
make forward reference table
# there is an array forward_references[], with one entry for
# each token in the input; forward_references[i] gives the
# token number where a token sequence starts with the same
# hash value as the one starting at i
compare various files to find runs
delete forward reference table
pass2, find newline positions of found similarities
pass3, print the similarities
pass1, read the files:
for each file
divide the text into tokens
store all tokens except newlines in Token_Array and try to
keep a record of the newline positions
make forward reference table:
# there are two independent hash functions, hash1() and hash2().
# hash1(i) gives the hash value of the token sequence starting at i
# likewise for hash2(i)
set up the forward references using the last_index table:
# there is an array last_index[], with one entry for each
# possible hash value; last_index[i] gives the position in
# forward_references[] at which i was most recently
# encountered as a hash value
for each file
for all positions in file except the last MinRunSize
set forward_references[] and update last_index[]
use hash2() to clean out matches:
for all tokens
find first token in chain with same hash2 code
short-circuit forward reference to it
compare:
for all new files
for all texts it must be compared to
for all positions in the new file
for all positions in the text
for ever increasing sizes
try to match and keep the best
try to match and keep the best:
# using forward_references[], we find a list of positions in
# which a matching token sequence will start;
# scanning this list, we measure the maximum length of the
# match and add the longest match to the run collection
pass2, find positions of found runs:
for all files:
sort the positions in the runs
# we scan the pos list and the file in parallel
for all positions inside this file
if it matches a token position in a run
record line number
pass3, print the similarities:
for all runs
# a run consists of two chunks
open the files that hold the chunks and position them
at the beginning of the chunk
display the chunks

83
core/sim/sim_3_01/ToDo Normal file
View File

@@ -0,0 +1,83 @@
- tx_pos is a pig. struct texts are in principle immutable objects, except that
lists of positions are attached to it. These lists (and that is the root of
the problem )are accessed from two points, one in pass2 to fill in NL
positions and in pass3 to report them. This makes it impossible to write
const struct text in most places. Maybe there should be a superstruct
struct super_text {
const struct *st_text;
struct position *st_pos;
}
Not elegant either.
- some size_t are sizes, others are positions, indexes
- start,limit -> start,length
- Bool.h
- report runs as '... <matching text only> ...' (proper name for Retrieve_Runs())
- unify idf2token() in *lang.l
- get rid of static forward references to routines; occurrences:
egr static *.c | grep "("
1 compare.c
7 hash.c
2 pass1.c
4 pass2.c
12 pass3.c
1 stream.c
3 text.c
- lex_nl_cnt counts from 1; this requires small, complicating adjustments
Done ================================================================
+ command line parameter consistency
+ make sim_text case-indifferent?
+ sortlist.bdy by split-merge
+ get rid of the nl_buff mechanism. No, use 16 bits line length.
+ in hash.c, size_t -> uint64_t? No, unit32_t is just as good.
+ / misinterpreted by shell; | alternative
+ register - removed
+ Run hashing OK: average chain length = 1.5, for sim-ing the sources of MCD2
+ Idf hashing OK: smooth distribution when sim-ing the sources of MCD2
+ use two-byte tokens to obtain better resolution for sim_text and on -F option
and UTF-8 (Johnson, Benjamin (US - Chicago))
+ different defaults per program
+ cleaning up sim.c & names
+ Microsoft comment (// ... unescaped \n)
+ emails 2009-2011 (A = I answered, R= they replied)
+AR Marcus Brinkmann, separate letters
+AR Scott Kuhl, percentages
+AR Yaroslav Halchenko, identifying non-existent lines
+A Rumen Stefanov, UTF-8
+A Jonathan Martin, UTF-8
+AR UTF-8 (Johnson, Benjamin (US - Chicago))
+ better structure between X.h and X.c
+ clean-up language.h and its sub-class algollike.h
+ warning in README to correct for non-MSDOS
Rejected ================================================================
X remove Miranda
X Mon Apr 11 13:23:41 1994: sim_orca
X Thu May 13 23:02:46 1993: sim ook voor C++ en Ada
X db_ not protected by #ifdef but by compilation to a call to an (empty) routine
1. not conspicuous enough in the code; 2. impairs efficiency

1
core/sim/sim_3_01/VERSION Normal file
View File

@@ -0,0 +1 @@
3.0

25
core/sim/sim_3_01/add_run.c Normal file
View File

@@ -0,0 +1,25 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: add_run.c,v 2.18 2016-06-12 13:00:04 dick Exp $
*/
#include "sim.h"
#include "text.h"
#include "runs.h"
#include "percentages.h"
#include "options.h"
#include "add_run.h"
/* Sends the run info to add_to_percentages or to add_to_runs. */
void
add_run(struct text *txt0, size_t i0,
struct text *txt1, size_t i1,
size_t size
) {
if (is_set_option('p')) {
add_to_percentages(txt0, txt1, size);
}
else {
add_to_runs(txt0, i0, txt1, i1, size);
}
}

22
core/sim/sim_3_01/add_run.h Normal file
View File

@@ -0,0 +1,22 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: add_run.h,v 1.5 2016-04-10 09:11:11 dick Exp $
*/
/* Interface between front-end and back-end: all information about
runs passes through add_run(). Its parameters are the two chunks,
each identified by their struct text and the position of the common
segment in Token_Array[], and the number of tokens in the common
segment.
The routine add_run() constructs a run with the given properties and
adds it to 'runs.[ch]' or 'percentages.[ch]'.
*/
extern void add_run(
struct text *txt0, /* text of first chunk */
size_t i0, /* chunk position in Token_Array[] */
struct text *txt1, /* text of second chunk */
size_t i1, /* chunk position in Token_Array[] */
size_t size /* number of tokens in the chunk */
);

137
core/sim/sim_3_01/algollike.c Normal file
View File

@@ -0,0 +1,137 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: algollike.c,v 2.11 2016-05-13 19:00:51 dick Exp $
*/
#include "sim.h"
#include "options.h"
#include "token.h"
#include "algollike.h"
/* Arrays for fast identification tests for tokens. Each token is
identified by its position in the set + 1. For example, if tk is
the n-th Opener, openers[Token2int(tk)] == n+1.
*/
static char non_finals[N_REGULAR_TOKENS];
static char non_initials[N_REGULAR_TOKENS];
static char openers[N_REGULAR_TOKENS];
static char closers[N_REGULAR_TOKENS];
/* Init_Language */
static void
cvt2bittable(const Token *tl, char bt[]) {
/* assumes bt[] is cleared */
int i;
int cnt = 1;
for (i = 0; !Token_EQ(tl[i], No_Token); i++) {
int index = Token2int(tl[i]);
if (index < 0 || index >= N_REGULAR_TOKENS)
fatal("internal error: bad Token list");
bt[index] = cnt++;
}
}
void
Init_Algol_Language(
const Token Non_Finals[],
const Token Non_Initials[],
const Token Openers[],
const Token Closers[]
) {
/* convert the token sets to bitmaps for speed-up */
cvt2bittable(Non_Initials, non_initials);
cvt2bittable(Non_Finals, non_finals);
cvt2bittable(Openers, openers);
cvt2bittable(Closers, closers);
}
/* May_Be_Start_Of_Run */
static int
pos_in_set(const char set[], const Token tk) {
if (!is_regular_token(tk)) return 0;
return set[Token2int(tk)];
}
int
May_Be_Start_Of_Algol_Run(const Token tk) {
return pos_in_set(non_initials, tk) == 0;
}
/* Best_Run_Size */
static size_t
largest_routine(const Token *tk_array, size_t size) {
/* Returns the size of the longest sequence starting at
tk_array[0] and not containing unbalanced parentheses.
Does not check the nesting of the parentheses, but then,
sim is syntax-free anyway.
*/
size_t mrb_size = 0; /* most recent balancing size */
size_t pos;
int i;
int balance_count[N_REGULAR_TOKENS];
/* Overkill: only a fraction of the tokens are balancers; oh well. */
int n_imbalances;
/* clear administration */
n_imbalances = 0;
for (i = 0; i < N_REGULAR_TOKENS; i++) {
balance_count[i] = 0;
}
/* scan tk_array[] and see how far we get */
for (pos = 0; pos < size; pos++) {
Token tk = tk_array[pos];
int pp; /* parenthesis position */
/* account for openers */
if ((pp = pos_in_set(openers, tk))) {
if (balance_count[pp] == 0) {
/* about to create an imbalance */
n_imbalances++;
}
balance_count[pp]++;
}
/* account for closers */
if ((pp = pos_in_set(closers, tk))) {
if (balance_count[pp] == 0) {
/* this is one Closer too many */
return mrb_size;
}
balance_count[pp]--;
if (balance_count[pp] == 0) {
/* we just cleared an imbalance */
n_imbalances--;
}
}
if (n_imbalances == 0) {
/* register the balance point */
mrb_size = pos + 1;
}
}
return mrb_size;
}
size_t
Best_Algol_Run_Size(const Token *tk_array, size_t size) {
/* Checks the run starting at tk_array[0] with length size for
acceptability in the language. Cuts from the end if necessary
and returns the accepted length, which may be zero.
*/
if (is_set_option('f')) {
/* reduce to a routine-like form first */
size = largest_routine(tk_array, size);
}
while ( /* there is trailing garbage */
size != 0 && pos_in_set(non_finals, tk_array[size-1])
) {
/* remove it */
size--;
}
return size;
}

32
core/sim/sim_3_01/algollike.h Normal file
View File

@@ -0,0 +1,32 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: algollike.h,v 1.7 2013-04-28 16:30:40 dick Exp $
*/
/* The class Algollike is a subclass of Language. It implements
the routines
void Init_Algol_Language()
int May_Be_Start_Of_Algol_Run() and
size_t Best_Algol_Run_Size()
for ALGOL-like languages, languages in which it is meaningful and
useful to isolate function bodies. These routines can be used in
Init_Language(), May_Be_Start_Of_Run(), and Best_Run_Size(), required
by language.h .
It requires the user to define four token sets, represented as
Token set[] and terminated by No_Token:
Token Non_Finals[] tokens that may not end a chunk
Token Non_Initials[] tokens that may not start a chunk
Token Openers[] openers of parentheses that must
balance in functions
Token Closers[] the corresponding closers,
in the same order
These must be passed to Init_Algol_Language(), in the above order.
*/
extern void Init_Algol_Language(
const Token Non_Finals[], const Token Non_Initials[],
const Token Openers[], const Token Closers[]
); /* note the order of the arguments: Non_Finals ~ Openers, etc. */
extern int May_Be_Start_Of_Algol_Run(Token ch);
extern size_t Best_Algol_Run_Size(const Token *str, size_t size);

86
core/sim/sim_3_01/any_int.c Normal file
View File

@@ -0,0 +1,86 @@
/* This file is part of the module ANY_INT.
Written by Dick Grune, dick@dickgrune.com
$Id: any_int.c,v 1.4 2017-01-22 14:50:00 Gebruiker Exp $
*/
#include "any_int.h"
#define N_INDEPENDENT_CALLS 12
#define MAX_ANY_UINT_DIGITS 40 /* good for 128 bits, including sign */
/*Library module source prelude */
#undef _ANY_UINT_CODE_
#ifndef lint
#define _ANY_UINT_CODE_
#endif
#ifdef LIB
#define _ANY_UINT_CODE_
#endif
#ifdef _ANY_UINT_CODE_
/* Library module source code */
/* circular list of buffers */
static char buff[N_INDEPENDENT_CALLS][MAX_ANY_UINT_DIGITS+1];
static int next_buff_cnt = 0;
static char *
next_buff(void) {
if (next_buff_cnt == N_INDEPENDENT_CALLS) next_buff_cnt = 0;
return buff[next_buff_cnt++];
}
static const char *
int2string(vlong_uint val, int neg, int size) {
char *res = next_buff() + MAX_ANY_UINT_DIGITS; /* end of new buffer */
*res = '\0'; /* insert EOS */
/* protect size */
if (size < 0 || size > MAX_ANY_UINT_DIGITS) size = 0;
do { /* one decimal character, the first always */
*--res = "0123456789ABCDEF"[val % 10];
size--;
val = val / 10;
} while (val > 0);
if (neg) {
*--res = '-';
size--;
}
while (size > 0) { /* fill up to size */
*--res = ' ';
size--;
}
return res;
}
const char * /* transient * N_INDEPENDENT_CALLS */
any_int2string(vlong_int val, int size) {
int neg = 0;
if (val < 0) {
val = - val;
neg = 1;
}
return int2string((vlong_uint)val, neg, size);
}
const char * /* transient * N_INDEPENDENT_CALLS */
any_uint2string(vlong_uint val, int size) {
return int2string(val, 0, size);
}
/* End library module source code */
#endif /* _ANY_UINT_CODE_ */
#ifdef lint
static void
satisfy_lint(void *x) {
any_int2string(0, 0);
any_uint2string(0, 0);
satisfy_lint(x);
}
#endif /* lint */

35
core/sim/sim_3_01/any_int.h Normal file
View File

@@ -0,0 +1,35 @@
/* This file is part of the module ANY_INT.
Written by Dick Grune, dick@dickgrune.com
$Id: any_int.h,v 1.4 2017-01-22 14:50:00 Gebruiker Exp $
*/
#ifndef _ANY_INT_H_
#define _ANY_INT_H_
/* Printing size_t and very long ints.
Printing integers using *printf requires specifying the format, which
requires knowing the exact nature of the integer. But this is not always
the case, f.e with size_t or extra-long integers for the accumulation of
size_t values. Some systems use %z as a dedicated format to print size_t,
but this is not portable since not all compilers know it.
These problems are solved by introducing the type vlong_[u]int (see below),
defined as the largest [unsigned] machine int type on the system,
and routines to convert these to string.
The resulting string is transient, but up to N_INDEPENDENT_CALLS calls
can be used simultaneously.
Since the value is passed to the conversion routines as a typed parameter
the C compiler does the conversion (actually widening) for you.
*/
/* Public entries */
typedef long long int vlong_int; /* largest int in the system */
typedef unsigned long long int vlong_uint; /* largest uint in the system */
/* transient * N_INDEPENDENT_CALLS */
extern const char *any_int2string(vlong_int val, int size);
extern const char *any_uint2string(vlong_uint val, int size);
#endif /* _ANY_INT_H_ */

370
core/sim/sim_3_01/c++lang.l Normal file
View File

@@ -0,0 +1,370 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: c++lang.l,v 2.2 2017-03-19 09:23:19 dick Exp $
*/
/*
C++ language front end for the similarity tester.
Based on C language front end written by Dick Grune <dick@cs.vu.nl>
Author: Dick Grune <dick@cs.vu.nl>
Modified by: Evin Murphy, UCD Dublin, Ireland
*/
#include "options.h"
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "idf.h"
#include "lex.h"
#include "lang.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent data */
/* Data for module idf */
static const struct idf ppcmd[] = {
{"define", META('d')},
{"elif", META('e')},
{"else", META('E')},
{"endif", META('n')},
{"error", META('r')},
{"if", META('i')},
{"ifdef", META('I')},
{"ifndef", META('x')},
{"include", MTCT('I')},
{"line", META('l')},
{"pragma", META('p')},
{"undef", META('u')}
};
static const struct idf reserved[] = {
{"and", NORM('a')},
{"and_eq", NORM('A')},
{"asm", CTRL('A')},
{"auto", META('a')},
{"bitand", NORM('b')},
{"bitor", NORM('B')},
{"bool", CTRL('B')},
{"break", META('b')},
{"case", NORM('c')},
{"char", NORM('C')},
{"class", CTRL('C')},
{"const", META('c')},
{"const_cast", META('C')},
{"continue", MTCT('C')},
{"default", NORM('d')},
{"delete", NORM('D')},
{"do", CTRL('D')},
{"double", META('D')},
{"dynamic_cast", MTCT('D')},
{"else", NORM('e')},
{"enum", NORM('E')},
{"explicit",CTRL('E')},
{"extern", MTCT('E')},
{"false", NORM('f')},
{"float", NORM('F')},
{"for", CTRL('F')},
{"friend", META('f')},
{"goto", NORM('g')},
{"if", NORM('i')},
{"inline", NORM('I')},
{"int", CTRL('I')},
{"long", NORM('l')},
{"mutable", NORM('m')},
{"namespace", NORM('n')},
{"new", NORM('N')},
{"not", CTRL('N')},
{"not_eq", META('N')},
{"operator", NORM('o')},
{"or", NORM('O')},
{"or_eq", CTRL('O')},
{"private", NORM('p')},
{"protected", NORM('P')},
{"public", CTRL('P')},
{"register", No_Token},
{"reinterpret_cast", NORM('r')},
{"return", NORM('R')},
{"short", NORM('s')},
{"signed", NORM('S')},
{"sizeof", CTRL('S')},
{"static", META('s')},
{"static_cast", META('S')},
{"struct", MTCT('S')},
{"switch", META('w')},
{"template", NORM('t')},
{"this", NORM('T')},
{"throw", CTRL('T')},
{"true", META('t')},
{"try", META('T')},
{"typedef", MTCT('T')},
{"typeid", NORM('y')},
{"typename", NORM('z')},
{"union", NORM('u')},
{"unsigned", NORM('U')},
{"using", CTRL('U')},
{"virtual", NORM('v')},
{"void", NORM('V')},
{"while", NORM('w')},
{"xor", NORM('x')},
{"xor_eq", NORM('X')}
};
/* Special treatment of identifiers */
static Token
idf2token(int hashing) {
Token tk;
tk = idf_in_list(yytext, reserved, sizeof reserved, IDF);
if (Token_EQ(tk, IDF) && hashing) {
/* return a one-Token hash code */
tk = idf_hashed(yytext);
}
return tk;
}
/* Token sets for module algollike */
static const Token Non_Finals[] = {
IDF, /* identifier */
NORM('{'),
NORM('('),
NORM('a'), /* and */
NORM('A'), /* and_eq */
CTRL('A'), /* asm */
META('a'), /* auto */
NORM('b'), /* bitand */
NORM('B'), /* bitor */
CTRL('B'), /* bool */
META('b'), /* break */
NORM('c'), /* case */
NORM('C'), /* char */
CTRL('C'), /* class */
META('c'), /* const */
META('C'), /* const_cast */
MTCT('C'), /* continue */
NORM('d'), /* default */
NORM('D'), /* delete */
CTRL('D'), /* do */
META('D'), /* double */
MTCT('D'), /* dynamic_cast */
NORM('e'), /* else */
NORM('E'), /* enum */
CTRL('E'), /* explicit */
MTCT('E'), /* extern */
NORM('F'), /* float */
CTRL('F'), /* for */
META('f'), /* friend */
NORM('g'), /* goto */
NORM('i'), /* if */
NORM('I'), /* inline */
CTRL('I'), /* int */
NORM('l'), /* long */
NORM('m'), /* mutable */
NORM('n'), /* namespace */
NORM('N'), /* new */
CTRL('N'), /* not */
META('N'), /* not_eq */
NORM('o'), /* operator */
NORM('O'), /* or */
CTRL('O'), /* or_eq */
NORM('p'), /* private */
NORM('P'), /* protected */
CTRL('P'), /* public */
NORM('r'), /* reinterpret_cast */
NORM('R'), /* return */
NORM('s'), /* short */
NORM('S'), /* signed */
CTRL('S'), /* sizeof */
META('s'), /* static */
META('S'), /* static_cast */
MTCT('S'), /* struct */
META('w'), /* switch */
NORM('t'), /* template */
CTRL('T'), /* throw */
META('T'), /* try */
MTCT('T'), /* typedef */
NORM('y'), /* typeid */
NORM('z'), /* typename */
NORM('u'), /* union */
NORM('U'), /* unsigned */
CTRL('U'), /* using */
NORM('v'), /* virtual */
NORM('V'), /* volatile */
CTRL('V'), /* void */
NORM('w'), /* while */
NORM('x'), /* xor */
NORM('X'), /* xor_eq */
No_Token
};
static const Token Non_Initials[] = {
NORM(')'),
NORM('}'),
NORM(';'),
No_Token
};
static const Token Openers[] = {
NORM('{'),
NORM('('),
NORM('['),
No_Token
};
static const Token Closers[] = {
NORM('}'),
NORM(')'),
NORM(']'),
No_Token
};
/* Language-dependent code */
const char *Subject = "C++ programs";
void
Init_Language(void) {
Init_Algol_Language(Non_Finals, Non_Initials, Openers, Closers);
}
int
May_Be_Start_Of_Run(Token ch) {
return May_Be_Start_Of_Algol_Run(ch);
}
size_t
Best_Run_Size(const Token *str, size_t size) {
return Best_Algol_Run_Size(str, size);
}
%}
%option noyywrap
%Start Comment
Layout ([ \t\r\f])
ASCII95 ([\040-\176])
Digit ([0-9a-fA-F])
UniCode (\\u{Digit}{Digit}{Digit}{Digit})
AnyQuoted ((\\.)|{UniCode})
StrChar ([^\"\n\\]|{AnyQuoted})
ChrChar ([^\'\n\\]|{AnyQuoted})
StartComment ("/*")
EndComment ("*/")
SafeComChar ([^*\n])
UnsafeComChar ("*")
SingleLineCom ("//".*)
Idf ([A-Za-z][A-Za-z0-9_]*)
%%
{StartComment} {
/* We do not have one single pattern to match a comment
(although one can be written), for two reasons.
The matched string might overflow lex-internal buffers
like yysbuf and yytext; and the pattern would be very
complicated and impair maintainability.
So we break up the string into safe chunks and keep
track of where we are in a start condition <Comment>.
*/
BEGIN Comment;
}
<Comment>{SafeComChar}+ { /* safe comment chunk */
}
<Comment>{UnsafeComChar} { /* unsafe char, read one by one */
}
<Comment>"\n" { /* to break up long comments */
return_eol();
}
<Comment>{EndComment} { /* end-of-comment */
BEGIN INITIAL;
}
{SingleLineCom}"\n" { /* single-line comment */
return_eol();
}
\"{StrChar}*\" { /* strings */
return_ch('"');
}
\'{ChrChar}+\' { /* characters */
return_ch('\'');
}
^#{Layout}*include.* { /* ignore #include lines */
}
^#{Layout}*{Idf} { /* a preprocessor line */
char *idf = yytext+1;
/* skip layout in front of preprocessor identifier */
while (*idf == ' ' || *idf == '\t') {
idf++;
}
return_tk(idf_in_list(idf, ppcmd, sizeof ppcmd, NORM('#')));
}
(0x)?{Digit}+("l"|"L")? { /* numeral, passed as an identifier */
return_tk(IDF);
}
{Idf}/"(" { /* identifier in front of ( */
Token tk;
tk = idf2token(is_set_option('F'));
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
{Idf} { /* identifier */
Token tk;
tk = idf2token(0 /* no hashing */);
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
\; { /* semicolon, conditionally ignored */
if (is_set_option('f')) return_ch(yytext[0]);
}
\n { /* count newlines */
return_eol();
}
{Layout} { /* ignore layout */
}
{ASCII95} { /* copy other text */
return_ch(yytext[0]);
}
. { /* count non-ASCII chars */
lex_non_ascii_cnt++;
}
%%
/* More language-dependent code */
void
yystart(void) {
BEGIN INITIAL;
}

282
core/sim/sim_3_01/clang.l Normal file
View File

@@ -0,0 +1,282 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: clang.l,v 2.22 2017-03-19 09:23:19 dick Exp $
*/
/*
C language front end for the similarity tester.
Author: Dick Grune <dick@cs.vu.nl>
*/
#include "options.h"
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "idf.h"
#include "lex.h"
#include "lang.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent data */
/* Data for module idf */
static const struct idf ppcmd[] = {
{"define", META('d')},
{"else", META('e')},
{"endif", META('E')},
{"if", META('i')},
{"ifdef", META('I')},
{"ifndef", META('x')},
{"include", MTCT('I')},
{"line", META('l')},
{"undef", META('u')}
};
static const struct idf reserved[] = {
{"auto", NORM('a')},
{"break", NORM('b')},
{"case", NORM('c')},
{"char", NORM('C')},
{"continue", CTRL('C')},
{"default", NORM('d')},
{"do", NORM('D')},
{"double", CTRL('D')},
{"else", NORM('e')},
{"enum", NORM('E')},
{"extern", CTRL('E')},
{"float", NORM('f')},
{"for", NORM('F')},
{"goto", NORM('g')},
{"if", NORM('i')},
{"int", NORM('I')},
{"long", NORM('l')},
{"register", No_Token},
{"return", NORM('r')},
{"short", NORM('s')},
{"sizeof", NORM('S')},
{"static", CTRL('S')},
{"struct", META('s')},
{"switch", META('S')},
{"typedef", NORM('t')},
{"union", NORM('u')},
{"unsigned", NORM('U')},
{"void", No_Token},
{"while", NORM('w')}
};
/* Special treatment of identifiers */
static Token
idf2token(int hashing) {
Token tk;
tk = idf_in_list(yytext, reserved, sizeof reserved, IDF);
if (Token_EQ(tk, IDF) && hashing) {
/* return a one-Token hash code */
tk = idf_hashed(yytext);
}
return tk;
}
/* Token sets for module algollike */
static const Token Non_Finals[] = {
IDF, /* identifier */
NORM('{'),
NORM('('),
NORM('a'), /* auto */
NORM('b'), /* break */
NORM('c'), /* case */
NORM('C'), /* char */
CTRL('C'), /* continue */
NORM('d'), /* default */
NORM('D'), /* do */
CTRL('D'), /* double */
NORM('E'), /* enum */
CTRL('E'), /* extern */
NORM('f'), /* float */
NORM('F'), /* for */
NORM('g'), /* goto */
NORM('i'), /* if */
NORM('I'), /* int */
NORM('l'), /* long */
NORM('r'), /* return */
NORM('s'), /* short */
CTRL('S'), /* static */
META('s'), /* struct */
META('S'), /* switch */
NORM('t'), /* typedef */
NORM('u'), /* union */
NORM('U'), /* unsigned */
NORM('w'), /* while */
No_Token
};
static const Token Non_Initials[] = {
NORM(')'),
NORM('}'),
NORM(';'),
No_Token
};
static const Token Openers[] = {
NORM('{'),
NORM('('),
NORM('['),
No_Token
};
static const Token Closers[] = {
NORM('}'),
NORM(')'),
NORM(']'),
No_Token
};
/* Language-dependent code */
const char *Subject = "C programs";
void
Init_Language(void) {
Init_Algol_Language(Non_Finals, Non_Initials, Openers, Closers);
}
int
May_Be_Start_Of_Run(Token ch) {
return May_Be_Start_Of_Algol_Run(ch);
}
size_t
Best_Run_Size(const Token *str, size_t size) {
return Best_Algol_Run_Size(str, size);
}
%}
%option noyywrap
%Start Comment
Layout ([ \t\r\f])
ASCII95 ([\040-\176])
AnyQuoted (\\.)
StrChar ([^\"\n\\]|{AnyQuoted})
ChrChar ([^\'\n\\]|{AnyQuoted})
StartComment ("/*")
EndComment ("*/")
SafeComChar ([^*\n])
UnsafeComChar ("*")
MSComment ("//"{MSCommentChar}*)
MSCommentChar ([^\n])
Digit ([0-9a-fA-F])
Idf ([A-Za-z][A-Za-z0-9_]*)
%%
{StartComment} {
/* We do not have one single pattern to match a comment
(although one can be written), for two reasons.
The matched string might overflow lex-internal buffers
like yysbuf and yytext; and the pattern would be very
complicated and impair maintainability.
So we break up the string into safe chunks and keep
track of where we are in a start condition <Comment>.
*/
BEGIN Comment;
}
<Comment>{SafeComChar}+ { /* safe comment chunk */
}
<Comment>{UnsafeComChar} { /* unsafe char, read one by one */
}
<Comment>"\n" { /* to break up long comments */
return_eol();
}
<Comment>{EndComment} { /* end-of-comment */
BEGIN INITIAL;
}
{MSComment} { /* ignore */
}
\"{StrChar}*\" { /* strings */
return_ch('"');
}
\'{ChrChar}+\' { /* characters */
return_ch('\'');
}
^#{Layout}*include.* { /* ignore #include lines */
}
^#{Layout}*{Idf} { /* a preprocessor line */
char *idf = yytext+1;
/* skip layout in front of preprocessor identifier */
while (*idf == ' ' || *idf == '\t') {
idf++;
}
return_tk(idf_in_list(idf, ppcmd, sizeof ppcmd, NORM('#')));
}
(0x)?{Digit}+("l"|"L")? { /* numeral, passed as an identifier */
return_tk(IDF);
}
{Idf}/"(" { /* identifier in front of ( */
Token tk;
tk = idf2token(is_set_option('F'));
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
{Idf} { /* identifier */
Token tk;
tk = idf2token(0 /* no hashing */);
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
\; { /* semicolon, conditionally ignored */
if (is_set_option('f')) return_ch(yytext[0]);
}
\n { /* count newlines */
return_eol();
}
{Layout} { /* ignore layout */
}
{ASCII95} { /* copy other text */
return_ch(yytext[0]);
}
. { /* count non-ASCII chars */
lex_non_ascii_cnt++;
}
%%
/* More language-dependent code */
void
yystart(void) {
BEGIN INITIAL;
}

406
core/sim/sim_3_01/compare.c Normal file
View File

@@ -0,0 +1,406 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: compare.c,v 2.38 2016-08-07 15:24:17 dick Exp $
*/
#include "sim.h"
#include "text.h"
#include "token.h"
#include "tokenarray.h"
#include "hash.h"
#include "language.h"
#include "options.h"
#include "add_run.h"
#include "compare.h"
#include "debug.par"
/* The overall structure of the routine Compare_Files() is:
for all new files
for all texts it must be compared to
for all positions in the new file
for all positions in the text
for ever increasing sizes
try to match and keep the best
*/
/* LOCAL VARIABLES */
static size_t beginning_of_text;
static size_t beginning_of_old_text;
static size_t end_of_text;
/* RANGE HANDLING */
struct range {
size_t rg_start;
size_t rg_limit;
int rg_sticky;
};
static int
in_range(size_t i, const struct range *rg) {
if (rg->rg_start <= rg->rg_limit) {
/* one single range */
return (rg->rg_start <= i && i < rg->rg_limit);
} else {
/* looped-around range */
return (rg->rg_start <= i && i < end_of_text
|| beginning_of_text <= i && i < rg->rg_limit);
}
}
static int
is_empty_range(const struct range *rg) {
return rg->rg_start == rg->rg_limit;
}
/* COMPARE FILES */
static void compare_one_text(int n, struct range *rg);
static void compare_one_on_one(int n, int m, struct range *rg);
static size_t lcs(
struct text *txt0, size_t i0, struct range *rg,
struct text **tx_bp, size_t *i_bp
);
void
Compare_Files(void) {
int n;
for ( /* all new texts */
n = 0; n < Number_of_New_Texts; n++
) {
struct range range;
/* construct default range */
beginning_of_text = Text[0].tx_start;
beginning_of_old_text = Text[Number_of_New_Texts-1].tx_limit;
end_of_text = Text[Number_of_Texts-1].tx_limit;
range.rg_start = Text[n].tx_start + 1;
range.rg_limit = end_of_text;
range.rg_sticky = 0;
/* update range for options */
if (is_set_option('a')) {
/* all text */
range.rg_start = Text[n].tx_start + 1;
range.rg_limit = Text[n].tx_start;
range.rg_sticky = 1;
}
if (is_set_option('S')) {
/* old text only */
range.rg_start = beginning_of_old_text;
range.rg_limit = end_of_text;
range.rg_sticky = 0;
}
if (is_set_option('s')) {
if ( /* n in range */
range.rg_start == Text[n].tx_start + 1
) { /* take it out */
range.rg_start = Text[n].tx_limit;
range.rg_sticky = 0;
}
}
if (is_empty_range(&range)) continue;
/* compare the files */
if (is_set_option('e')) {
/* over the range in steps of one */
int m;
for (m = n; m < Number_of_Texts; m++) {
compare_one_on_one(n, m, &range);
}
for (m = 0; m < n; m++) {
compare_one_on_one(n, m, &range);
}
}
else {
/* the whole range in one action */
compare_one_text(n, &range);
}
}
}
static void
compare_one_on_one(
int n, /* index of text to be compared */
int m, /* index of text to be compared to */
struct range *rg /* pointer to search range */
) {
const struct text *txt1 = &Text[m];
if (!in_range(txt1->tx_start+1, rg)) return;
/* construct private range consisting of Text[m] */
struct range range_m;
range_m.rg_start = txt1->tx_start;
range_m.rg_limit = txt1->tx_limit;
range_m.rg_sticky = 0;
/* compare Text[n] and Text[m] */
compare_one_text(n, &range_m);
}
static void
compare_one_text(
int n, /* index of text to be compared */
struct range *rg /* pointer to search range */
) {
struct text *txt0 = &Text[n];
size_t i0 = txt0->tx_start;
#ifdef DB_COMP
fprintf(Debug_File, "compare_one_text(%s", txt0->tx_fname);
fprintf(Debug_File,
", i0 = %d, rg_start = %d, rg_limit = %d, sticky = %d)\n",
i0, rg->rg_start, rg->rg_limit, rg->rg_sticky);
#endif
while ( /* there is room for a run */
i0 + Min_Run_Size <= txt0->tx_limit
) {
if (!May_Be_Start_Of_Run(Token_Array[i0])) {
/* no point in looking; try the next token */
i0++;
} else {
/* see if there really is a run */
struct text *txt_run;
size_t i_run;
size_t run_size =
lcs(txt0, i0, rg, &txt_run, &i_run);
if (run_size) {
/* run found; enter it */
#ifdef DB_COMP
fprintf(Debug_File,
"add_run(%s, %d, %s, %d, %d)\n",
txt0->tx_fname, i0,
txt_run->tx_fname, i_run, run_size);
#endif
add_run(txt0, i0, txt_run, i_run, run_size);
/* and skip it */
i0 += run_size;
}
else {
/* we try our luck at the next token */
i0++;
}
}
if (rg->rg_sticky) {
/* drag rg->rg_start along */
rg->rg_start = i0 - 1;
}
}
}
#ifdef DB_COMP_2
static void
fprint_tokens(FILE *o, size_t p0, size_t p1) {
fprintf(o, "\n \"");
while (p0 < p1) {
fprintf(o, " %3d", Token_Array[p0]);
p0++;
}
fprintf(o, "\"");
}
#endif
static size_t
first_forward_ref_for(size_t i0, const struct range *rg) {
size_t res = Forward_Reference(i0, i0);
while (res && !in_range(res, rg)) {
res = Forward_Reference(res, i0);
}
return res;
}
static size_t
lcs( struct text *txt0, /* text to be compared */
size_t i0, /* starting pos. in txt0 */
struct range *rg, /* search range */
/* two output parameters, set if return value > 0: */
struct text **tx_bp, /* output, text of best run */
size_t *i_bp /* starting pos. in text of best run */
) {
/* Finds the longest common substring (not subsequence) in:
txt0, starting precisely at i0 and
all the text in the search range rg.
Writes the position in tx_bp and i_bp and returns the size.
Returns 0 if no common substring is found.
*/
size_t i1;
size_t size_best = 0;
if (!(txt0->tx_start <= i0 && i0 < txt0->tx_limit))
fatal("i0 not inside txt0");
#ifdef DB_COMP
fprintf(Debug_File,
"lcs(i0 = %d, rg_start = %d, rg_limit = %d), FWR[i0] = %d, ffr[i0] = %d\n",
i0, rg->rg_start, rg->rg_limit,
Forward_Reference(i0, i0),
first_forward_ref_for(i0, rg)
);
#endif
for ( i1 = first_forward_ref_for(i0, rg);
i1 && in_range(i1, rg);
i1 = Forward_Reference(i1, i0)
) {
/* i1 is always on the forward reference chain of i0 */
/* Find the text txt1 into which i1 points. */
#if 0
struct text *txt1 = txt0;
while (i1 < txt1->tx_start) {
txt1--;
}
while (i1 >= txt1->tx_limit) {
txt1++;
}
#else
struct text *txt1;
{ struct text *txt_b = &Text[0];
struct text *txt_e = &Text[Number_of_Texts-1];
while (txt_b != txt_e) {
struct text *txt_m = txt_b + (txt_e-txt_b) / 2;
if (i1 < txt_m->tx_limit) {
txt_e = txt_m;
} else {
txt_b = txt_m + 1;
}
}
txt1 = txt_b;
}
#endif
if (!(txt1->tx_start <= i1 && i1 < txt1->tx_limit))
fatal("i1 not inside txt1");
#ifdef DB_COMP
fprintf(Debug_File, "for i1: %s, i0=%d,%s, i1=%d\n",
txt0->tx_fname, i0, txt1->tx_fname, i1);
#endif
size_t better_size = (size_best ? size_best+1 : Min_Run_Size);
/* Are we looking at something better than we have got? */
{ /* we compare backwards from the end of
a putative better match starting at i1
*/
size_t j0 = i0 + better_size - 1;
size_t j1 = i1 + better_size - 1;
#ifdef DB_COMP
fprintf(Debug_File, "init: j0 = %d, j1 = %d\n", j0, j1);
#endif
/* would there be room for a better match? */
if ( /* j0 still inside txt0 */
j0 < txt0->tx_limit
&& /* j1 still inside txt1 */
j1 < txt1->tx_limit
&& /* better 0 and better 1 don't overlap */
(j0 < i1 || j1 < i0)
) {
/* yes, there is room enough for a match */
/* see if the text matches for at least
better_size tokens
*/
/* since we have perfect forward references and
chack backwards, we do not have to check the
last Min_Run_Size tokens:
*/
size_t cnt = better_size - Min_Run_Size;
#ifdef DB_COMP
fprintf(Debug_File,
"init verification: cnt = %d", cnt);
#ifdef DB_COMP_2
/* we don't want this all the time under
DB_COMP, but we want it linted
*/
fprint_tokens(Debug_File, i0, i0+better_size);
fprint_tokens(Debug_File, i1, i1+better_size);
#endif
fprintf(Debug_File, "\n");
#endif
while ( cnt
&& Token_EQ(Token_Array[j0],
Token_Array[j1])
) {
cnt--, j0--, j1--;
}
#ifdef DB_COMP
fprintf(Debug_File,
"end verification: cnt = %d\n", cnt);
#endif
if (cnt) {
/* not all tokens matched,
so forget it */
continue;
}
} else {
/* no, there is not enough room for a better
match, so forget it */
continue;
}
}
/* Yes, we are looking at a better match;
how long can we make it?
*/
size_t new_size = better_size;
{ /* extending forwards */
size_t j0 = i0 + better_size;
size_t j1 = i1 + better_size;
while ( /* j0 still inside txt0 */
j0 < txt0->tx_limit
&& /* j1 still inside txt1 */
j1 < txt1->tx_limit
&& /* j0 and j1 don't overlap */
(j0 < i1 || j1 < i0)
&& /* tokens are the same */
Token_EQ(Token_Array[j0], Token_Array[j1])
) {
j0++, j1++, new_size++;
}
}
#ifdef DB_COMP
fprintf(Debug_File,
"end forward extension: new_size = %d\n", new_size);
#endif
/* Offer the run to the Language Department which may
reject it or may cut its tail.
*/
new_size = Best_Run_Size(&Token_Array[i0], new_size);
if ( /* we still have an acceptable run */
new_size >= Min_Run_Size
&& /* it is still better than what we had */
new_size > size_best
) {
/* record it */
#ifdef DB_COMP
fprintf(Debug_File,
"possible run: %s, %d, %s, %d, %d\n",
txt0->tx_fname, i0,
txt1->tx_fname, i1, new_size);
#endif
*tx_bp = txt1;
*i_bp = i1;
size_best = new_size;
}
/* and see if it can be improved with a different i1 */
}
#ifdef DB_COMP
fprintf(Debug_File, "lcs out, size_best = %d\n",
size_best);
#endif
return size_best;
}

11
core/sim/sim_3_01/compare.h Normal file
View File

@@ -0,0 +1,11 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: compare.h,v 1.4 2016-04-10 11:19:47 dick Exp $
*/
/* Compares each new text to the appropriate texts.
Stores the runs found by passing them to add_run().
Runs contain references to positions in the input files.
*/
extern void Compare_Files(void);

115
core/sim/sim_3_01/debug.c Normal file
View File

@@ -0,0 +1,115 @@
/* This file is part of the debugging module DEBUG.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: debug.c,v 1.7 2014-09-25 06:58:25 Gebruiker Exp $
*/
#include <stdlib.h>
#include <unistd.h>
#include <ctype.h>
#include "debug.h"
/*Library module source prelude */
#undef _MODULE_CODE_
#ifndef lint
#define _MODULE_CODE_
#endif
#ifdef LIB
#define _MODULE_CODE_
#endif
#ifdef _MODULE_CODE_
/* Library module source code */
#ifdef DEBUG
static void
wr_char(char ch) {
write(2, &ch, 1);
}
static void
wr_num(int b,int v) {
if (v >= b) {
wr_num(b, v/b);
}
wr_char("0123456789ABCDEF"[v%b]);
}
static void
wr_str(const char *s) {
while (*s) {
wr_char(*s++);
}
}
void
wr_info(const char *s, int v) {
/* print the string */
if (s) {
int cnt = 0;
while (*s) {
int ch = *s++ &0377;
/* cut short a possibly corrupted string */
if (cnt++ > 50) {
wr_str("...");
break;
}
/* put not thy faith in chars, signed or unsigned */
if (isprint(ch)) {
wr_char(ch);
}
else {
switch (ch) {
case '\n': wr_str("\\n"); break;
case '\t': wr_str("\\t"); break;
case '\r': wr_str("\\r"); break;
case '\f': wr_str("\\f"); break;
default:
wr_char('\\');
wr_char(ch / 0100 % 010 + '0');
wr_char(ch / 010 % 010 + '0');
wr_char(ch / 01 % 010 + '0');
break;
}
}
}
}
else {
wr_str("<wr_info: NO STRING>");
}
/* print the value */
wr_char(' ');
if (v < 0) {
wr_char('-');
v = -v;
}
wr_num(10, v);
wr_char('\n');
}
#else
/*ARGSUSED*/
void
wr_info(const char *s, int v) {
}
#endif /* DEBUG */
/* End library module source code */
#endif /* _MODULE_CODE_ */
#ifdef lint
static void
satisfy_lint(void *x) {
wr_info((char *)x, 0);
satisfy_lint(x);
}
#endif /* lint */

28
core/sim/sim_3_01/debug.h Normal file
View File

@@ -0,0 +1,28 @@
/* This file is part of the debugging module DEBUG.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: debug.h,v 1.9 2014-09-25 06:58:26 Gebruiker Exp $
*/
#ifndef _DEBUG_H_
#define _DEBUG_H_
/****
The module DEBUG defines one routine,
extern void wr_info(const char *str, int val);
which, when compiled with a -DDEBUG option, writes the string str, a space
character, the value val in decimal, and a newline to standard error output
(file descriptor 2), without interfering with other program activities.
This allows debugging info to be obtained in the presence of sudden crashes
and other nefarious program activity.
Compiled without the -DDEBUG option wr_info does nothing. This allows easy
switching off of the debugging feature by recompiling debug.c.
****/
/* Public entries */
extern void wr_info(const char *s, int v);
#endif /* _DEBUG_H_ */

29
core/sim/sim_3_01/debug.par Normal file
View File

@@ -0,0 +1,29 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: debug.par,v 1.14 2016-07-29 20:41:32 dick Exp $
*/
#undef DB_HASH /* print hash value info */
#undef DB_FORW_REF /* check forward references */
#undef DB_FORW_REF_PRINT /* also print forward references */
#undef DB_COMP /* print comparison activity */
#undef DB_COMP_2 /* include token comparison */
#undef DB_TEXT /* print all text parts */
#undef DB_POS /* print positions in files */
#undef DB_NL_BUFF /* print the newline count buffer */
#undef DB_RUN /* print all run activity */
#undef DB_PERC /* print the percentage match list */
#if defined(lint)
#define DB_HASH
#define DB_FORW_REF
#define DB_FORW_REF_PRINT
#define DB_COMP
#define DB_COMP_2
#define DB_TEXT
#define DB_POS
#define DB_NL_BUFF
#define DB_RUN
#define DB_PERC
#endif

336
core/sim/sim_3_01/fname.c Normal file
View File

@@ -0,0 +1,336 @@
/* This file is part of the auxiliaries library.
Written by Dick Grune, dick@dickgrune.com
$Id: fname.c,v 1.8 2017-01-22 14:50:00 Gebruiker Exp $
*/
/* Support for UNICODE file names in Windows */
/* Two data types are involved in UNICODE file names in Windows:
UTF16 strings, the file names as stored by Windows, and
UTF8 strings, the names as they are displayed and stored.
The conversion between these two proceeds through CodePoints,
the 'real' values of the characters, of which UTF16 strings and
UTF8 strings are the compressed representations.
The module consists of two levels: a set of static routines
int UTF8_sequence_to_CodePoint(const UTF8 *s, UTF32 *cp),
int UTF16_sequence_to_CodePoint(const UTF16 *s, UTF32 *cp),
const UTF8 *CodePoint_to_UTF8_sequence(UTF32 c),
const Fchar *CodePoint_to_UTF16_sequence(UTF32 cp)
which convert from and to CodePoints; and two global routines
const char *Fname2str(const Fchar *fn),
const Fchar *str2Fname(const char *s)
which convert from Unicode file names to UTF-8 strings and vice versa.
Unicode file names are obtained by calls of Opendir() and Readdir(),
as defined in fname.h.
*/
#include "fname.h"
/*Library module source prelude */
#undef _FNAME_CODE_
#ifndef lint
#define _FNAME_CODE_
#endif
#ifdef LIB
#define _FNAME_CODE_
#endif
#ifdef _FNAME_CODE_
/* Library module source code */
#ifdef _UNICODE
typedef uint8_t UTF8;
typedef uint16_t UTF16;
typedef uint32_t UTF32;
#define BAD_CodePoint (UTF32)(-1)
/* mask of n left-aligned 1-s in a UTF8 */
#define LMASK(n) (((1<<(n))-1)<<(8-(n)))
/* mask of n right-aligned 1-s */
#define RMASK(n) ((1u<<(n))-1)
/* SEQUENCE -> CODEPOINT */
static int
nmb_leading_ones_in_UTF8(UTF8 c) {
int n = 0;
while (c&LMASK(1)) {
c <<= 1, n++;
}
return n;
}
static UTF32
get_UTF8_tail(const UTF8 *s, int n) {
UTF32 res = 0;
int i;
/* scoop up n UTF-8s */
for (i = 0; i < n; i++) {
if ((s[i]&LMASK(2)) != LMASK(1)) return BAD_CodePoint;
res = (res<<6) + (s[i]&RMASK(6));
}
return res;
}
static int /* number of UTF8s used; cp = BAD_CodePoint for error */
UTF8_sequence_to_CodePoint(const UTF8 *s, UTF32 *cp) {
UTF8 head = s[0];
int head_length = 1;
const UTF8 *tail = &s[1];
int tail_length;
UTF32 tail_value;
if ((head&LMASK(1)) == 0) {
*cp = head;
return head_length;
}
tail_length = nmb_leading_ones_in_UTF8(head) - 1;
if (tail_length < 1 || tail_length > 3) goto error;
tail_value = get_UTF8_tail(tail, tail_length);
if (tail_value == BAD_CodePoint) goto error;
*cp = ((head&RMASK(6-tail_length))<<(tail_length*6)) | tail_value;
return head_length+tail_length;
error: { int i = head_length; /* skip the head */
/* skip until new head */
while ((s[i]&LMASK(1)) != 0) {
i++;
}
*cp = BAD_CodePoint;
return i;
}
}
static int
is_in_BMP(UTF32 c) { /* Basic Multilingual Plane */
return c <= 0xD7FF || (0xE000 <= c && c < 0x10000);
}
static int
is_high_surrogate(UTF16 c) {
return 0xD800 <= c && c <= 0xDBFF;
}
static int
is_low_surrogate(UTF16 c) {
return 0xDC00 <= c && c <= 0xDFFF;
}
static int /* number of UTF16s used; cp = BAD_CodePoint for error */
UTF16_sequence_to_CodePoint(const UTF16 *s, UTF32 *cp) {
/* adapted from code from http://unicode.org/faq/utf_bom.html */
UTF32 plane_number;
UTF32 position;
if (is_in_BMP(s[0])) {
*cp = s[0];
return 1;
}
/* s[0:1] must be a surrogate pair */
if (!is_high_surrogate(s[0])) goto error;
if (!is_low_surrogate(s[1])) goto error;
/* get the plane number */
plane_number = (s[0] >> 6) & RMASK(5);
plane_number = plane_number + 1; /* to offset it from the BMP */
/* get the position in the plane */
position = ((s[0] & RMASK(6)) << 10) | (s[1] & RMASK(10));
/* combine them */
*cp = plane_number << 16 | position;
return 2;
error: { int i = 1; /* skip one UTF-16 */
/* skip until acceptable UTF-16 */
while (!is_in_BMP(s[i]) && !is_high_surrogate(s[0])) {
i++;
}
*cp = BAD_CodePoint;
return i;
}
}
/* CODEPOINT -> SEQUENCE */
static const UTF8 * /* transient */
CodePoint_to_UTF8_sequence(UTF32 c) {
/* adapted from code by user R on stackoverflow.com */
static UTF8 buff[6];
UTF8 *bp = buff;
if (c < 0x80) {
/* it fits in 7 bits */
*bp++ = (c>>0)&RMASK(7);
}
else if (c < 0x800) {
/* it fits in 11 bits */
*bp++ = 0xC0 | ((c>>6)&RMASK(5));
*bp++ = 0x80 | ((c>>0)&RMASK(6));
}
else if (c < 0x10000) {
/* it fits in 16 bits */
if (!is_in_BMP(c)) {
/* it is in the forbidden zone */
return NULL;
}
*bp++ = 0xE0 | ((c>>12)&RMASK(4));
*bp++ = 0x80 | ((c>>6)&RMASK(6));
*bp++ = 0x80 | ((c>>0)&RMASK(6));
}
else if (c < 0x110000) {
/* it fits in 21 bits */
*bp++ = 0xF0 | ((c>>18)&RMASK(3));
*bp++ = 0x80 | ((c>>12)&RMASK(6));
*bp++ = 0x80 | ((c>>6)&RMASK(6));
*bp++ = 0x80 | ((c>>0)&RMASK(6));
}
else return NULL;
*bp = '\0';
return buff;
}
static UTF16 * /* transient */
CodePoint_to_UTF16_sequence(UTF32 cp) {
/* adapted from code from http://unicode.org/faq/utf_bom.html */
static UTF16 res[3];
if (is_in_BMP(cp)) {
res[0] = cp;
res[1] = '\0';
return res;
}
if (cp >= 0x10000) {
UTF16 position = (UTF16) cp;
UTF16 plane_number = ((cp >> 16) & RMASK(5)) - 1;
res[0] = 0xD800 | (plane_number << 6) | (position >> 10);
res[1] = 0xDC00 | (position & RMASK(10));
res[2] = '\0';
return res;
}
else return NULL;
}
const char * /* transient */
Fname2str(const Fchar *fn) {
/* converts a Fchar (wchar_t) string to an UTF-8 string */
static UTF8 res[1024];
UTF8 *rp = &res[0];
int i = 0;
if (fn == NULL) return NULL;
while (fn[i]) {
UTF32 cp;
const UTF8 *p;
/* get Codepoint from one or two Fchar chars */
i += UTF16_sequence_to_CodePoint(&fn[i], &cp);
if (cp == BAD_CodePoint) goto error;
/* convert code point to UTF8 sequence */
p = CodePoint_to_UTF8_sequence(cp);
if (p == NULL) goto error;
/* append it to the output */
while (*p) {
*rp++ = *p++;
}
continue;
error:
*rp++ = '?';
}
*rp = '\0';
return (const char *)res;
}
const Fchar * /* transient */
str2Fname(const char *s) {
/* converts a possibly UTF-8 string to an Fchar (wchar_t) string */
static Fchar res[512];
Fchar *rp = &res[0];
int i = 0;
if (s == NULL) return NULL;
while (s[i]) {
UTF32 cp;
const Fchar *p;
/* get Codepoint from one to four UTF-8s */
i += UTF8_sequence_to_CodePoint((const UTF8 *)&s[i], &cp);
if (cp == BAD_CodePoint) goto error;
/* convert code point to UTF-16 sequence */
p = CodePoint_to_UTF16_sequence(cp);
if (p == NULL) goto error;
/* append it to the output */
while (*p) {
*rp++ = *p++;
}
continue;
error:
*rp++ = '?';
}
*rp = '\0';
return res;
}
/* OTHER UTF-16 ROUTINES */
int
Stat(const Fchar *fn, struct stat *st) {
/* why on earth does _wstat use a funny struct _stat ? */
return _wstat(fn, (struct _stat *)st);
}
FILE *
Fopen(const Fchar *fn, const char *rb) { /* stream is still char* */
Fchar fn_copy[512];
/* avoid possible transiency of fn */
Fnamecpy(fn_copy, fn);
return _tfopen(fn_copy, str2Fname(rb));
}
#endif /* _UNICODE */
/* End library module source code */
#endif /* _FNAME_CODE_ */
#ifdef lint
static void
satisfy_lint(void *x) {
/* lint cannot handle Fchar complications */
satisfy_lint(x);
}
#endif /* lint */

122
core/sim/sim_3_01/fname.h Normal file
View File

@@ -0,0 +1,122 @@
/* This file is part of the auxiliaries library.
Written by Dick Grune, dick@dickgrune.com
$Id: fname.h,v 1.13 2017-01-22 14:50:00 Gebruiker Exp $
*/
/* Support for UNICODE file names */
/*
To accommodate UNICODE file names on various platforms, this file defines
the types
Fchar file name character
Dir_t struct for accessing a directory
Dirent_t struct for accessing a directory entry
and the functions
Dir_t* Opendir(const Fchar*);
Dirent_t* Readdir(Dir_t*);
int Closedir(Dir_t*);
Fchar *Fnamecpy(Fchar *dest, Fchar *source);
Fchar *Fnamecat(Fchar*, const Fchar*);
int Fnamelen(const Fchar*);
int Fnamecmp(const Fchar*, const Fchar*);
int Stat(const Fchar *fn, struct stat *st);
FILE *Fopen(const Fchar *fn, const char *rb);
The stream is still char*!
int Fclose(FILE*);
const char *Fname2str(const Fchar *fn);
const Fchar *str2Fname(const char *s);
The result of these two routines is transient: is is good only until
the next call.
The only way to obtain a file name is through readdir; the command line
arguments are in ASCII. So a program can be adapted by replacing
DIR by Dir_t, and
struct dirent by Dirent_t.
Compiling and correcting using the above replacements until there are no
more errors or warnings will then yield an UTF-16 compatible program, as
far as the input is concerned. Output is done in UTF-8; there seems to be
no way to do output in UFT-16.
For details about UTF-16 see fname.c.
*/
#ifndef _FNAME_H_
#define _FNAME_H_
/* lint cannot handle the weird code Windows throws at it, so even under
Windows we claim to have UTF8
*/
#ifdef MSDOS
#define IS_UTF_16
#endif
#ifdef lint
#undef IS_UTF_16
#endif
#ifdef IS_UTF_16 /* file names in UTF-16 */
#define _UNICODE
#include <tchar.h>
#include <sys/stat.h>
#include <dirent.h>
/* Private entries */
typedef _TCHAR Fchar;
typedef _WDIR Dir_t;
typedef struct _tdirent Dirent_t;
/* Public entries */
#define Opendir _topendir
#define Closedir _tclosedir
#define Readdir _treaddir
#define Fnamecpy wcscpy
#define Fnamecat wcscat
#define Fnamelen (int)wcslen
#define Fnamecmp wcscmp
extern const char *Fname2str(const Fchar *fn); /* transient! */
extern const Fchar *str2Fname(const char *s); /* transient! */
extern int Stat(const Fchar *fn, struct stat *st);
extern FILE *Fopen(const Fchar *fn, const char *rb);/* stream is still char* */
#define Fclose fclose
#else /* not MSDOS */ /* file names are in UTF-8 */
#include <sys/stat.h>
#include <dirent.h>
#include <string.h>
/* life is simple */
/* Public entries */
typedef char Fchar;
#define Fnamecpy strcpy
#define Fnamecat strcat
#define Fnamelen strlen
#define Fnamecmp strcmp
#define Fname2str(fn) (fn)
#define str2Fname(s) (s)
#define Stat(fn,st) stat(fn,st)
typedef DIR Dir_t;
typedef struct dirent Dirent_t;
#define Opendir opendir
#define Closedir closedir
#define Readdir readdir
#define Fopen fopen
#define Fclose fclose
#endif /* MSDOS */
#endif /* _FNAME_H_ */

434
core/sim/sim_3_01/hash.c Normal file
View File

@@ -0,0 +1,434 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: hash.c,v 2.39 2017-02-04 16:58:54 dick Exp $
*/
/* Text is compared by comparing every substring to all substrings
to the right of it; this process is in essence quadratic. However,
only substrings of length at least 'Min_Run_Size' are of interest,
which gives us the possibility to speed up this process by using
a hash table.
For every position p in the text, we construct an index table entry
forward_reference[p] which gives the next position in the text
at which a run of Min_Run_Size tokens starts that has the same
hash code, as calculated by hash1(). If there is no such run,
the index is 0.
To construct this array, we use a hash table latest_index[] whose size
is a prime and which is about the same size as the text array.
The hash table latest_index[] is set up such that latest_index[i] is the
index of the latest token with hash_code i, or 0 if there is none.
See Make_Forward_References().
The forward references produced this way are not perfect, due to hashing
coincidences. A second sweep (make_forward_references_perfect()) makes
them perfect by doing a full comparison on Min_Run_Size tokens.
For the LaTeX sources of our book Modern Compiler Design, 2nd Ed. this
reduced the total forward chain length from 103555 to 345, as
determined by db_forward_reference_check().
The forward references can be checked with db_forward_reference_check(),
which also collects statistics.
*/
#include <stdio.h>
#include <stdint.h>
#include "system.par"
#include "debug.par"
#include "sim.h"
#include "text.h"
#include "Malloc.h"
#include "any_int.h"
#include "token.h"
#include "language.h"
#include "token.h"
#include "tokenarray.h"
#include "options.h"
#include "hash.h"
/* MAIN ENTRIES */
static size_t *forward_reference; /* to be filled by Malloc() */
static size_t n_forward_references;
static void make_forward_references_using_hash(void);
static void make_forward_references_perfect(void);
static void make_chains_circular(void);
#ifdef DB_FORW_REF
static void db_forward_reference_check(const char *);
static void db_print_forward_references(void);
#endif /* DB_FORW_REF */
void
Make_Forward_References(void) {
/* Constructs the forward references table.
*/
n_forward_references = Token_Array_Length();
forward_reference =
(size_t *)Calloc(n_forward_references, sizeof (size_t));
make_forward_references_using_hash();
make_forward_references_perfect();
if (is_set_option('a')) {
make_chains_circular();
}
#ifdef DB_FORW_REF_PRINT
db_print_forward_references();
#endif /* DB_FORW_REF_PRINT */
}
size_t
Forward_Reference(size_t i, size_t i0) {
if (i == 0 || i >= n_forward_references) {
fatal("internal error, bad forward reference");
}
size_t new_i = forward_reference[i];
size_t res = new_i == 0 || new_i == i0 /*circular*/ ? 0 : new_i;
return res;
}
void
Free_Forward_References(void) {
Free(forward_reference);
}
/* HASHING */
static size_t *latest_index;
static size_t latest_index_table_size;
/* The prime numbers of the form 4 * i + 3 for some i, all greater
than twice the previous one and smaller than 2^40 (for now).
*/
static const uint64_t prime[] = {
#if 0
3,
7,
19,
43,
103,
211,
431,
863,
1747,
3499,
7019,
#endif
14051,
28111,
56239,
112507,
225023,
450067,
900139,
1800311,
3600659,
7201351,
14402743,
28805519,
57611039,
115222091,
230444239,
460888499,
921777067,
1843554151,
UINT64_C (3687108307),
UINT64_C (7374216631),
UINT64_C (14748433279),
UINT64_C (29496866579),
UINT64_C (58993733159),
UINT64_C (117987466379),
UINT64_C (235974932759),
UINT64_C (471949865531),
UINT64_C (943899731087)
/* 2^40= 1099511627776 */
};
static void
init_hash_table(void) {
int n;
/* find the ideal hash table size */
n = 0;
while (prime[n] < Token_Array_Length()) {
n++;
/* this will always terminate, if prime[] is large enough */
}
/* see if we can allocate that much space, and if not, step down */
latest_index = 0;
while ( /* we have not yet obtained our array */
!latest_index
&& /* and there is still a (prime) size left to try */
n >= 0
) {
latest_index_table_size = prime[n];
latest_index = (size_t *)
TryCalloc(latest_index_table_size, sizeof (size_t));
n--;
}
if (!latest_index) {
fatal("out of memory: no room for hash table");
}
}
static void
make_forward_references_using_hash(void) {
int n;
init_hash_table();
/* Set up the forward references using the latest_index[] hash table. */
for (n = 0; n < Number_of_Texts; n++) {
const struct text *txt = &Text[n];
size_t j;
uint32_t hash = 0;
#define Left_Circular_32(i, s) (((i) << (s)) | ((i) >> (32-(s))))
#define SHIFT (5)
for (j = txt->tx_start; j < txt->tx_limit; j++) {
if ( /* we have a complete hash value */
j - txt->tx_start >= Min_Run_Size
) { /* remove the oldest token */
Token oldest_token =
Token_Array[j - Min_Run_Size];
int oldest_shift =
((Min_Run_Size-1) * SHIFT) % 32;
hash ^=
Left_Circular_32(oldest_token, oldest_shift);
}
/* Circular left shift */
hash = Left_Circular_32(hash, SHIFT);
/* Add new token */
hash ^= Token_Array[j];
/* If have we assembled a complete hash value now,
the corresponding run would start at
j - (Min_Run_Size - 1). For it to be valid it
should start at or after txt->tx_start, so we would
like to write the test
j - (Min_Run_Size - 1) >= txt->tx_start. However,
the type of this computation is size_t, which is
unsigned, and j - (Min_Run_Size - 1) may be negative,
so we code instead:
*/
if (j - txt->tx_start < (Min_Run_Size - 1)) {
/* no */
continue;
}
/* We now have the complete hash value for a run ending
at j and can safely compute j - (Min_Run_Size - 1).
*/
size_t run_start = j - (Min_Run_Size - 1);
/* Can the run be useful? */
if (!May_Be_Start_Of_Run(Token_Array[run_start]))
continue; /* no*/
/* the hash value is used here for an index */
size_t h = hash % latest_index_table_size;
if (latest_index[h]) {
forward_reference[latest_index[h]] = run_start;
}
/*latest_index[h] = j;*/
latest_index[h] = run_start;
}
}
Free(latest_index);
#ifdef DB_FORW_REF
db_forward_reference_check("first hashing");
#endif /* DB_FORW_REF */
}
static void
make_chains_circular(void) {
size_t i;
/* Make the chains circular, by a slightly quadratic algorithm. */
for (i = 0; i+Min_Run_Size < Token_Array_Length(); i++) {
if (!forward_reference[i]) continue;
size_t j = i;
while (forward_reference[j]) {
size_t j1 = forward_reference[j];
if (j1 < j) break; /* has already been treated */
j = j1;
}
if (forward_reference[j] == 0 && j != i) {
/* tie it back to the beginning of the chain */
forward_reference[j] = i;
}
}
}
static int
is_eq_min_run(const Token *p, const Token *q) {
/* a full comparison for the tertiary sweep */
size_t n;
for (n = 0; n < Min_Run_Size; n++) {
if (!Token_EQ(p[n], q[n])) return 0;
}
return 1;
}
static void
make_forward_references_perfect(void) {
size_t i;
/* Simulate a perfect hash by doing a full comparison
over Min_Run_Size, for gathering statistics.
*/
for (i = 0; i+Min_Run_Size < Token_Array_Length(); i++) {
size_t j = i;
while ( /* there is still a forward reference */
(j = forward_reference[j])
&& /* it does not match over Min_Run_Size */
!is_eq_min_run(&Token_Array[i], &Token_Array[j])
) {
/* continue searching */
}
/* short-circuit forward reference to it, or to zero */
forward_reference[i] = j;
}
/* now we have perfect forward references */
#ifdef DB_FORW_REF
db_forward_reference_check("full Min_Run_Size comparison");
#endif /* DB_FORW_REF */
}
#ifdef DB_FORW_REF
static void
db_print_forward_references(void) {
/* also determines the lengths of the chains, for statistics */
size_t n;
size_t n_frw_chains = 0;
size_t tot_frwc_len = 0;
size_t *print_loc_of =
(size_t *)Calloc(Token_Array_Length(), sizeof (size_t));
size_t *number_of_chains_of_length =
(size_t *)Calloc(Token_Array_Length(), sizeof (size_t));
/* print the references */
for (n = 1; n < Token_Array_Length(); n++) {
size_t fw = forward_reference[n];
if (fw == 0) continue;
/* we have a chain */
fprintf(Debug_File, "FWR[%s]:", any_uint2string(n, 0));
/* is it old? */
if (print_loc_of[n]) {
fprintf(Debug_File, " see %s\n",
any_uint2string(print_loc_of[n], 0));
continue;
}
/* no, we have the beginning of a new chain */
size_t count = 0;
do {
count++;
fprintf(Debug_File, " %s",
any_uint2string(fw, 0));
print_loc_of[fw] = n;
fw = forward_reference[fw];
} while(fw && fw != n); /* continuing and not circular */
if (fw) { /* circular */
fprintf(Debug_File, " C");
count++;
}
n_frw_chains++;
tot_frwc_len += count;
number_of_chains_of_length[count]++;
fprintf(Debug_File, "\n");
}
/* print the chain lengths */
for (n = 1; n < Token_Array_Length(); n++) {
if (number_of_chains_of_length[n]) {
fprintf(Debug_File, "length[%d]:\t%d\n",
n, number_of_chains_of_length[n]);
}
}
fprintf(Debug_File,
"text length = %s, # forward chains = %s, av. frw chain length = %.2f\n\n",
any_uint2string(Token_Array_Length(), 0),
any_uint2string(n_frw_chains, 0),
(n_frw_chains ? 1.0 * tot_frwc_len / n_frw_chains : 0.0)
);
Free(number_of_chains_of_length);
Free(print_loc_of);
}
static void
db_frw_chain(size_t n, char *crossed_out) {
if (forward_reference[n] == 0) {
fprintf(Debug_File,
">>>> db_frw_chain() forward_reference[n] == 0 <<<<\n"
);
return;
}
size_t n_entries = 0;
size_t fw;
for (fw = n; fw; fw = forward_reference[fw]) {
if (crossed_out[fw]) {
fprintf(Debug_File,
">>>> error: forward references cross <<<<\n"
);
}
n_entries++;
crossed_out[fw] = 1;
}
#ifdef DB_FORW_REF_PRINT
fprintf(Debug_File, "chain_start = %s, n_entries = %s\n",
any_uint2string(n, 0), any_uint2string(n_entries, 0));
#endif /* DB_FORW_REF_PRINT */
}
static void
db_forward_reference_check(const char *msg) {
/* Each forward_reference[n] starts in principle a new
chain, and these chains never touch each other.
We check this property by marking the positions in each
chain in an array; if we meet a marked entry while
following a chain, it must have been on an earlier chain
and we have an error.
*/
size_t n;
char *crossed_out = (char *)Calloc(Token_Array_Length(), sizeof (char));
fprintf(Debug_File, "\n\n**** DB_FORWARD_REFERENCES, %s ****\n", msg);
fprintf(Debug_File, "latest_index_table_size = %s\n",
any_uint2string(latest_index_table_size, 0));
if (forward_reference[0]) {
fprintf(Debug_File,
">>>> forward_reference[0] is not zero <<<<\n"
);
}
for (n = 1; n < Token_Array_Length(); n++) {
if (forward_reference[n] && !crossed_out[n]) {
/* start of a new chain */
db_frw_chain(n, crossed_out);
}
}
#ifdef DB_FORW_REF_PRINT
db_print_forward_references();
#endif /* DB_FORW_REF_PRINT */
Free(crossed_out);
}
#endif /* DB_FORW_REF */

13
core/sim/sim_3_01/hash.h Normal file
View File

@@ -0,0 +1,13 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: hash.h,v 1.5 2016-04-27 12:59:11 dick Exp $
*/
/* Creating and consulting forward_reference[], used to speed up
the Longest Substring Allgorithm.
*/
extern void Make_Forward_References(void);
extern void Free_Forward_References(void);
/* with circularity check: */
extern size_t Forward_Reference(size_t i, size_t i0);

87
core/sim/sim_3_01/idf.c Normal file
View File

@@ -0,0 +1,87 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: idf.c,v 2.19 2015-01-17 10:20:40 dick Exp $
*/
#include <string.h>
#include "system.par"
#include "token.h"
#include "idf.h"
Token
idf_in_list(
const char *str,
const struct idf list[],
size_t listsize,
Token default_token
) {
int first = 0;
int last = (int) (listsize / sizeof (struct idf)) - 1;
while (first < last) {
int middle = (first + last) / 2;
if (strcmp(str, list[middle].id_tag) > 0) {
first = middle + 1;
}
else {
last = middle;
}
}
return (strcmp(str, list[first].id_tag) == 0
? list[first].id_tr
: default_token
);
}
#define HASH(h,ch) (((h) * 8209) + (ch)*613)
Token
idf_hashed(const char *str) {
int32 h = 0;
/* let's be careful about ranges; if done wrong it's hard to debug */
while (*str) {
int ch = *str++ & 0377;
/* ignore spaces in spaced words */
if (ch == ' ') continue;
/* -1 <= h <= 2^31-1 */
h = HASH(h, ch);
/* -2^31 <= h <= 2^31-1 */
if (h < 0) {
/* -2^31 <= h <= -1 */
h += 2147483647; /* 2^31-1 */
/* -1 <= h <= 2^31-2 */
}
else {
/* 0 <= h <= 2^31-1 */
}
/* -1 <= h <= 2^31-1 */
}
/* -1 <= h <= 2^31-1 */
if (h < 0) {
/* h = -1 */
h = 0;
}
/* 0 <= h <= 2^31-1 */
h %= (N_TOKENS - N_REGULAR_TOKENS - 1);
/* 0 <= h < N_TOKENS - N_REGULAR_TOKENS - 1 */
h += N_REGULAR_TOKENS;
/* N_REGULAR_TOKENS <= h < N_TOKENS - 1 */
return int2Token(h);
/* this avoids the regular tokens and End_Of_Line */
}
void
lower_case(char *str) {
char *s;
for (s = str; *s; s++) {
if ('A' <= *s && *s <= 'Z') {
*s += (-'A' + 'a');
}
}
}

35
core/sim/sim_3_01/idf.h Normal file
View File

@@ -0,0 +1,35 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: idf.h,v 2.13 2016-07-31 18:55:44 dick Exp $
*/
/* Idf module:
Token idf_in_list(
const char *str,
const struct idf list[],
size_t listsize,
Token default_token
);
looks up a keyword in a list of keywords l, represented as an
array of struct idf, and returns its translation as a token;
dflt is returned if the keyword is not found.
Token idf_hashed(char *str);
returns a token unequal to No_Token or End_Of_Line, derived
from str through hashing
*/
/* the struct for keywords etc. */
struct idf {
char *id_tag; /* an interesting identifier */
Token id_tr; /* with its one-Token translation */
};
/* public functions */
extern Token idf_in_list(
const char *str,
const struct idf list[],
size_t listsize,
Token default_token
);
extern Token idf_hashed(const char *str);
extern void lower_case(char *str);

291
core/sim/sim_3_01/javalang.l Normal file
View File

@@ -0,0 +1,291 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: javalang.l,v 1.16 2017-03-19 09:23:19 dick Exp $
*/
/*
Java language front end for the similarity tester.
Author: Dick Grune <dick@cs.vu.nl>
*/
#include "options.h"
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "idf.h"
#include "lex.h"
#include "lang.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent data */
static const struct idf reserved[] = {
{"abstract", NORM('a')},
{"boolean", NORM('b')},
{"break", NORM('B')},
{"byte", CTRL('B')},
{"case", NORM('c')},
{"catch", NORM('C')},
{"char", CTRL('C')},
{"class", META('c')},
{"continue", META('C')},
{"default", NORM('d')},
{"do", NORM('D')},
{"double", CTRL('D')},
{"else", NORM('e')},
{"extends", NORM('E')},
{"false", NORM('g')}, /* Boolean literal */
{"final", NORM('f')},
{"finally", NORM('F')},
{"float", CTRL('F')},
{"for", META('f')},
{"if", NORM('i')},
{"implements", NORM('I')},
{"import", CTRL('I')},
{"instanceof", META('i')},
{"int", META('I')},
{"interface", MTCT('I')},
{"long", NORM('l')},
{"native", NORM('n')},
{"new", NORM('N')},
{"null", CTRL('N')}, /* null literal */
{"package", NORM('p')},
{"private", NORM('P')},
{"protected", CTRL('P')},
{"public", META('p')},
{"return", NORM('r')},
{"short", NORM('s')},
{"static", NORM('S')},
{"super", CTRL('S')},
{"switch", META('s')},
{"synchronized",META('S')},
{"this", NORM('t')},
{"throw", NORM('T')},
{"throws", CTRL('T')},
{"true", META('t')}, /* Boolean literal */
{"void", NORM('v')},
{"volatile", NORM('V')},
{"while", NORM('w')}
};
/* Special treatment of identifiers */
static Token
idf2token(int hashing) {
Token tk;
tk = idf_in_list(yytext, reserved, sizeof reserved, IDF);
if (Token_EQ(tk, IDF) && hashing) {
/* return a one-Token hash code */
tk = idf_hashed(yytext);
}
return tk;
}
/* Token sets for module algollike */
const Token Non_Finals[] = {
IDF, /* identifier */
NORM('{'),
NORM('('),
NORM('a'), /* abstract */
NORM('b'), /* boolean */
NORM('B'), /* break */
CTRL('B'), /* byte */
NORM('c'), /* case */
NORM('C'), /* catch */
CTRL('C'), /* char */
META('c'), /* class */
META('C'), /* continue */
NORM('d'), /* default */
NORM('D'), /* do */
CTRL('D'), /* double */
NORM('e'), /* else */
NORM('E'), /* extends */
NORM('f'), /* final */
NORM('F'), /* finally */
CTRL('F'), /* float */
META('f'), /* for */
NORM('i'), /* if */
NORM('I'), /* implements */
CTRL('I'), /* import */
META('i'), /* instanceof */
META('I'), /* int */
MTCT('I'), /* interface */
NORM('l'), /* long */
NORM('n'), /* native */
NORM('N'), /* new */
NORM('p'), /* package */
NORM('P'), /* private */
CTRL('P'), /* protected */
META('p'), /* public */
NORM('r'), /* return */
NORM('s'), /* short */
NORM('S'), /* static */
CTRL('S'), /* super */
META('s'), /* switch */
META('S'), /* synchronized */
NORM('T'), /* throw */
CTRL('T'), /* throws */
NORM('v'), /* void */
NORM('V'), /* volatile */
NORM('w'), /* while */
No_Token
};
const Token Non_Initials[] = {
NORM(')'),
NORM('}'),
NORM(';'),
No_Token
};
const Token Openers[] = {
NORM('{'),
NORM('('),
NORM('['),
No_Token
};
const Token Closers[] = {
NORM('}'),
NORM(')'),
NORM(']'),
No_Token
};
/* Language-dependent code */
const char *Subject = "Java programs";
void
Init_Language(void) {
Init_Algol_Language(Non_Finals, Non_Initials, Openers, Closers);
}
int
May_Be_Start_Of_Run(Token ch) {
return May_Be_Start_Of_Algol_Run(ch);
}
size_t
Best_Run_Size(const Token *str, size_t size) {
return Best_Algol_Run_Size(str, size);
}
%}
%option noyywrap
%Start Comment
Layout ([ \t\r\f])
ASCII95 ([\040-\176])
Digit ([0-9a-fA-F])
UniCode (\\u{Digit}{Digit}{Digit}{Digit})
AnyQuoted ((\\.)|{UniCode})
StrChar ([^\"\n\\]|{AnyQuoted})
ChrChar ([^\'\n\\]|{AnyQuoted})
StartComment ("/*")
EndComment ("*/")
SafeComChar ([^*\n])
UnsafeComChar ("*")
SingleLineCom ("//".*)
Idf ([A-Za-z][A-Za-z0-9_]*)
%%
{StartComment} {
/* We do not have one single pattern to match a comment
(although one can be written), for two reasons.
The matched string might overflow lex-internal buffers
like yysbuf and yytext; and the pattern would be very
complicated and overtax lex.
So we break up the string into safe chunks and keep
track of where we are in a start condition <Comment>.
*/
BEGIN Comment;
}
<Comment>{SafeComChar}+ { /* safe comment chunk */
}
<Comment>{UnsafeComChar} { /* unsafe char, read one by one */
}
<Comment>"\n" { /* to break up long comments */
return_eol();
}
<Comment>{EndComment} { /* end-of-comment */
BEGIN INITIAL;
}
{SingleLineCom}"\n" { /* single-line comment */
return_eol();
}
\"{StrChar}*\" { /* strings */
return_ch('"');
}
\'{ChrChar}+\' { /* characters */
return_ch('\'');
}
(0x)?{Digit}+("l"|"L")? { /* numeral, passed as an identifier */
return_tk(IDF);
}
"import"{Layout}[^;]*; { /* import statement; ignore */
}
{Idf}/"(" { /* identifier in front of ( */
Token tk;
tk = idf2token(is_set_option('F'));
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
{Idf} { /* identifier */
Token tk;
tk = idf2token(0 /* no hashing */);
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
\; { /* semicolon, conditionally ignored */
if (is_set_option('f')) return_ch(yytext[0]);
}
\n { /* count newlines */
return_eol();
}
{Layout} { /* ignore layout */
}
{ASCII95} { /* copy other text */
return_ch(yytext[0]);
}
. { /* count non-ASCII chars */
lex_non_ascii_cnt++;
}
%%
/* More language-dependent code */
void
yystart(void) {
BEGIN INITIAL;
}

49
core/sim/sim_3_01/lang.c Normal file
View File

@@ -0,0 +1,49 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: lang.c,v 2.9 2016-05-13 19:00:52 dick Exp $
*/
/*
This is a dummy implementation of the module 'lang'.
Its actual implementation derives from one of the *lang.l files.
*/
#include <stdio.h>
#include <stdlib.h>
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "idf.h"
#include "lex.h"
#include "lang.h"
FILE *yyin;
int
yylex(void) {
abort();
#ifdef lint
(void)May_Be_Start_Of_Algol_Run(0);
(void)Best_Algol_Run_Size(0, 0);
(void)idf_in_list(0, 0, 0, 0);
(void)idf_hashed(0);
(void)lower_case(0);
#endif
return 0;
}
void
yystart(void) {
abort();
#ifdef lint
Init_Algol_Language(0, 0, 0, 0);
#endif
}
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;

29
core/sim/sim_3_01/lang.h Normal file
View File

@@ -0,0 +1,29 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: lang.h,v 1.9 2016-04-27 19:09:48 dick Exp $
*/
/*
The *lang.l files provide two interfaces:
language.[ch] static data about the language
lang.[ch] dynamic data about the input file's content
This is lang.[ch].
*/
/*
The abstract module 'lang' declares the names that provide access to
the lowest-level token routines and data.
The corresponding definitions must be provided by the *lang.l file.
There is a dummy implementation lang.c, to satisfy the module
consistency checker check-ch.
*/
extern FILE *yyin;
extern int yylex(void);
extern void yystart(void);
extern Token lex_token; /* token produced, or End_Of_Line */
extern size_t lex_nl_cnt; /* line count */
extern size_t lex_tk_cnt; /* token position */
extern size_t lex_non_ascii_cnt; /* # of non-ASCII chars found */

37
core/sim/sim_3_01/language.c Normal file
View File

@@ -0,0 +1,37 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: language.c,v 2.4 2016-08-05 15:31:19 dick Exp $
*/
/*
This is a dummy implementation of the abstract class 'language', so
there will not be a language.o file.
The actual implementation is provided by Xlang.o deriving through
Xlang.c from the pertinent Xlang.l file.
*/
#include <stdio.h>
#include <stdlib.h>
#include "token.h"
#include "language.h"
const char *Subject;
void
Init_Language(void) {
abort();
}
int
May_Be_Start_Of_Run(Token ch) {
if (ch == ch) abort();
return 0;
}
size_t
Best_Run_Size(const Token *str, size_t size) {
if (str == str || size == size) abort();
return 0;
}

26
core/sim/sim_3_01/language.h Normal file
View File

@@ -0,0 +1,26 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: language.h,v 1.9 2016-04-27 19:09:48 dick Exp $
*/
/*
The *lang.l files provide two interfaces:
language.[ch] static data about the language
lang.[ch] dynamic data about the input file's content
This is language.[ch].
*/
/*
The abstract class 'language' declares the names that give access to
the properties of the language.
The corresponding definitions must be provided by the *lang.l file.
There is a dummy implementation language.c, to satisfy the module
consistency checker check-ch.
*/
extern const char *Subject;
extern void Init_Language(void);
extern int May_Be_Start_Of_Run(Token ch);
extern size_t Best_Run_Size(const Token *str, size_t size);

8
core/sim/sim_3_01/lex.c Normal file
View File

@@ -0,0 +1,8 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: lex.c,v 1.9 2012-06-08 16:04:28 dick Exp $
*/
/* The service macros for the *lang.l files do not require code */
#include "lex.h"

9
core/sim/sim_3_01/lex.h Normal file
View File

@@ -0,0 +1,9 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: lex.h,v 2.12 2012-09-30 11:55:19 dick Exp $
*/
/* Macros for the *lang.l files */
#define return_tk(tk) {lex_tk_cnt++; lex_token = (tk); return 1;}
#define return_ch(ch) {lex_tk_cnt++; lex_token = int2Token((int)(ch)); return 1;}
#define return_eol() {lex_nl_cnt++; lex_token = End_Of_Line; return 1;}

144
core/sim/sim_3_01/lisplang.l Normal file
View File

@@ -0,0 +1,144 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: lisplang.l,v 2.19 2017-03-19 09:23:19 dick Exp $
*/
/*
LISP language front end for the similarity tester.
Author: Gertjan Akkerman <akkerm@cs.vu.nl>
Date: Thu, 9 Apr 87 11:15:23 MDT
*/
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "lex.h"
#include "lang.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent data */
#include "idf.h"
static const struct idf reserved[] = {
{"append", NORM('a')},
{"append1", NORM('b')},
{"atom", NORM('t')},
{"car", NORM('h')},
{"cdr", NORM('t')},
{"cond", NORM('c')},
{"cons", NORM('s')},
{"defun", NORM('u')},
{"do", NORM('d')},
{"eq", NORM('e')},
{"equal", NORM('e')}, /* See eq */
{"for", NORM('f')},
{"if", NORM('i')},
{"list", NORM('l')},
{"nconc", NORM('n')},
{"rplaca", NORM('A')},
{"rplacd", NORM('D')}
};
/* Token sets for module algollike */
const Token Non_Finals[] = {
NORM('('),
NORM('['),
No_Token
};
const Token Non_Initials[] = {
NORM(')'),
NORM(']'),
No_Token
};
const Token Openers[] = {
NORM('('),
NORM('['),
No_Token
};
const Token Closers[] = {
NORM(')'),
NORM(']'),
No_Token
};
/* Language-dependent code */
const char *Subject = "Lisp programs";
void
Init_Language(void) {
Init_Algol_Language(Non_Finals, Non_Initials, Openers, Closers);
}
int
May_Be_Start_Of_Run(Token ch) {
return May_Be_Start_Of_Algol_Run(ch);
}
size_t
Best_Run_Size(const Token *str, size_t size) {
return Best_Algol_Run_Size(str, size);
}
%}
%option noyywrap
%Start Comment
Layout ([ \t\r\f])
ASCII95 ([\040-\176])
AnyQuoted (\\.)
StrChar ([^\"\n\\]|{AnyQuoted})
ChrChar ([^\'\\]|{AnyQuoted})
IdfChar ([-!#$%&*+,/0-9:;<=>?@A-Z\\^_`a-z{}~])
EscIdf (({IdfChar}|\\.)+)
QuotIdf ("|"[^\|\n]*"|")
Idf ({EscIdf}|{QuotIdf})
%%
";".*$ { /* comment */
}
\"{StrChar}*\" { /* strings */
return_ch('"');
}
{Idf} { /* identifier */
return_tk(idf_in_list(yytext, reserved, sizeof reserved, IDF));
}
\n { /* count newlines */
return_eol();
}
{Layout} { /* ignore layout */
}
{ASCII95} { /* copy other text */
return_ch(yytext[0]);
}
. { /* count non-ASCII chars */
lex_non_ascii_cnt++;
}
%%
/* More language-dependent code */
void
yystart(void) {
BEGIN INITIAL;
}

338
core/sim/sim_3_01/m2lang.l Normal file
View File

@@ -0,0 +1,338 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: m2lang.l,v 2.21 2017-03-19 09:23:19 dick Exp $
*/
/*
Modula-2 language front end for the similarity tester.
Author: Dick Grune <dick@cs.vu.nl>
*/
#include "options.h"
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "idf.h"
#include "lex.h"
#include "lang.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent data */
/* Most Modula-2 programs start with a number of IMPORTs that look
very similar from program to program. These are skipped by ignoring
the reserved words IMPLEMENTATION, DEFINITION, MODULE, IMPORT
and FROM, having a flag skip_imports, and start reacting only
at the first non-ignored reserved word.
Also, the nesting comments require a state variable.
*/
/* Additional state variables, set in yystart() */
static int skip_imports;
static int comment_level;
/* Data for module idf */
static const struct idf reserved[] = {
{"AND", NORM('&')},
{"ARRAY", NORM('A')},
{"BEGIN", NORM('{')},
{"BY", NORM('B')},
{"CASE", NORM('c')},
{"CONST", NORM('C')},
{"DEFINITION", No_Token},
{"DIV", NORM('/')},
{"DO", NORM('D')},
{"ELSE", NORM('e')},
{"ELSIF", NORM('e')},
{"END", NORM('}')},
{"EXIT", NORM('E')},
{"EXPORT", CTRL('E')},
{"FOR", NORM('F')},
{"FROM", No_Token},
{"IF", NORM('i')},
{"IMPLEMENTATION", No_Token},
{"IMPORT", No_Token},
{"IN", NORM('I')},
{"LOOP", NORM('l')},
{"MOD", NORM('%')},
{"MODULE", No_Token},
{"NOT", NORM('~')},
{"OF", No_Token},
{"OR", NORM('O')},
{"POINTER", NORM('p')},
{"PROCEDURE", NORM('P')},
{"QUALIFIED", NORM('q')},
{"RECORD", NORM('r')},
{"REPEAT", NORM('R')},
{"RETURN", CTRL('r')},
{"SET", NORM('s')},
{"THEN", No_Token},
{"TO", NORM('t')},
{"TYPE", NORM('T')},
{"UNTIL", NORM('u')},
{"VAR", NORM('v')},
{"WHILE", NORM('w')},
{"WITH", NORM('W')},
};
static const struct idf standard[] = {
{"ABS", META('a')},
{"ADDRESS", META('A')},
{"ALLOCATE", MTCT('A')},
{"BITSET", META('b')},
{"BOOLEAN", META('B')},
{"CAP", META('c')},
{"CARDINAL", META('C')},
{"CHAR", MTCT('C')},
{"CHR", META('x')},
{"DEALLOCATE", META('d')},
{"DEC", META('D')},
{"EXCL", META('e')},
{"FALSE", META('f')},
{"FLOAT", META('F')},
{"HALT", META('h')},
{"HIGH", META('H')},
{"INC", META('i')},
{"INCL", META('I')},
{"INTEGER", MTCT('I')},
{"LONGCARD", META('L')},
{"LONGINT", META('L')},
{"LONGREAL", META('L')},
{"MAX", META('m')},
{"MIN", META('M')},
{"NEWPROCESS", META('n')},
{"NIL", META('N')},
{"ODD", META('o')},
{"ORD", META('O')},
{"PROC", META('p')},
{"REAL", META('r')},
{"SIZE", META('s')},
{"SYSTEM", META('S')},
{"TRANSFER", META('t')},
{"TRUE", META('T')},
{"TRUNC", MTCT('T')},
{"VAL", META('v')},
{"WORD", META('w')}
};
/* Special treatment of identifiers */
static Token
idf2token(int hashing) {
Token tk;
/* the token can be on two lists, reserved and standard */
tk = idf_in_list(yytext, reserved, sizeof reserved, IDF);
/* is it one of the keywords to be ignored? */
if (Token_EQ(tk, No_Token)) return tk;
/* The statement below is a significant comment
on the value of state variables.
*/
if (!Token_EQ(tk, IDF)) {
/* reserved word, stop the skipping */
skip_imports = 0;
}
else {
/* it is an identifier but not a reserved word */
if (skip_imports) {
/* skip it */
tk = 0;
}
else {
/* look further */
tk = idf_in_list(yytext, standard, sizeof standard, IDF);
if (Token_EQ(tk, IDF) && hashing) {
/* return a one-Token hash code */
tk = idf_hashed(yytext);
}
}
}
return tk;
}
/* Token sets for module algollike */
const Token Non_Finals[] = {
IDF, /* identifier */
NORM('{'), /* also BEGIN */
NORM('('),
NORM('['),
NORM('A'), /* ARRAY */
NORM('c'), /* CASE */
NORM('C'), /* CONST */
NORM('E'), /* EXIT */
NORM('F'), /* FOR */
NORM('i'), /* IF */
NORM('l'), /* LOOP */
NORM('p'), /* POINTER */
NORM('P'), /* PROCEDURE */
NORM('r'), /* RECORD */
NORM('R'), /* REPEAT */
CTRL('R'), /* RETURN */
NORM('s'), /* SET */
NORM('T'), /* TYPE */
NORM('v'), /* VAR */
NORM('w'), /* WHILE */
NORM('W'), /* WITH */
No_Token
};
const Token Non_Initials[] = {
NORM('}'),
NORM(')'),
NORM(']'),
NORM(';'),
No_Token
};
const Token Openers[] = {
NORM('{'),
NORM('('),
NORM('['),
No_Token
};
const Token Closers[] = {
NORM('}'),
NORM(')'),
NORM(']'),
No_Token
};
/* Language-dependent code */
const char *Subject = "Modula-2 programs";
void
Init_Language(void) {
Init_Algol_Language(Non_Finals, Non_Initials, Openers, Closers);
}
int
May_Be_Start_Of_Run(Token ch) {
return May_Be_Start_Of_Algol_Run(ch);
}
size_t
Best_Run_Size(const Token *str, size_t size) {
return Best_Algol_Run_Size(str, size);
}
%}
%option noyywrap
%Start Comment
Layout ([ \t\r\f])
ASCII95 ([\040-\176])
AnyQuoted (\\.)
QuStrChar ([^\"\n\\]|{AnyQuoted})
ApoStrChar ([^\'\n\\]|{AnyQuoted})
StartComment ("(*")
EndComment ("*)")
SafeComChar ([^*\n])
UnsafeComChar ("*")
Digit ([0-9a-fA-F])
Idf ([A-Za-z][A-Za-z0-9_]*)
%%
{StartComment} { /* See clang.l */
/* Lex itself is incapable of handling Modula-2's
nested comments. So let's help it a bit.
*/
if (comment_level == 0) {
BEGIN Comment;
}
comment_level++;
}
<Comment>{SafeComChar}+ { /* safe comment chunk */
}
<Comment>{UnsafeComChar} { /* unsafe char, read one by one */
}
<Comment>"\n" { /* to break up long comments */
return_eol();
}
<Comment>{EndComment} { /* end-of-comment */
comment_level--;
if (comment_level == 0) {
BEGIN INITIAL;
}
}
\"{QuStrChar}*\" { /* quoted strings */
return_ch('"');
}
\'{ApoStrChar}*\' { /* apostrophed strings */
return_ch('"');
}
{Digit}+("B"|"C"|"H")? { /* numeral, passed as an identifier */
return_tk(IDF);
}
"END"{Layout}*{Idf} { /* ignore identifier after END */
Token tk =
idf_in_list("END", reserved, sizeof reserved, No_Token);
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
{Idf}/"(" { /* identifier in front of ( */
Token tk = idf2token(is_set_option('F')/* hashing option */);
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
{Idf} { /* identifier */
Token tk = idf2token(0 /* no hashing */);
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
"<>" { /* <>, special equivalence */
return_ch('#');
}
\; { /* semicolon, conditionally ignored */
if (is_set_option('f')) return_ch(yytext[0]);
}
\n { /* count newlines */
return_eol();
}
{Layout} { /* ignore layout */
}
{ASCII95} { /* copy other text */
if (!skip_imports) return_ch(yytext[0]);
}
. { /* count non-ASCII chars */
lex_non_ascii_cnt++;
}
%%
/* More language-dependent code */
void
yystart(void) {
skip_imports = 1;
comment_level = 0;
BEGIN INITIAL;
}

152
core/sim/sim_3_01/miralang.l Normal file
View File

@@ -0,0 +1,152 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: miralang.l,v 1.13 2017-03-19 09:23:20 dick Exp $
*/
/*
Miranda language front end for the similarity tester.
Author: Emma Norling (ejn@cs.mu.oz.au)
Date: Nov 1998
*/
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "lex.h"
#include "lang.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent data */
#include "idf.h"
static const struct idf reserved[] = {
{"abstype", NORM('a')},
{"bool", NORM('b')},
{"char", NORM('c')},
{"const", META('c')},
{"div", NORM('d')},
{"False", NORM('F')},
{"if", NORM('i')},
{"mod", NORM('m')},
{"num", NORM('n')},
{"otherwise", NORM('o')},
{"readvals", NORM('r')},
{"show", NORM('s')},
{"sys_message", META('s')},
{"True", NORM('T')},
{"type", NORM('t')},
{"where", NORM('w')},
{"with", META('w')}
};
/* Token sets for module algollike */
const Token Non_Finals[] = {
NORM('('),
NORM('['),
NORM('='),
No_Token
};
const Token Non_Initials[] = {
NORM(')'),
NORM(']'),
No_Token
};
const Token Openers[] = {
NORM('('),
NORM('['),
NORM('='),
No_Token
};
const Token Closers[] = {
NORM(')'),
NORM(']'),
No_Token
};
/* Language-dependent code */
const char *Subject = "Miranda programs";
void
Init_Language(void) {
Init_Algol_Language(Non_Finals, Non_Initials, Openers, Closers);
}
int
May_Be_Start_Of_Run(Token ch) {
return May_Be_Start_Of_Algol_Run(ch);
}
size_t
Best_Run_Size(const Token *str, size_t size) {
return Best_Algol_Run_Size(str, size);
}
%}
%option noyywrap
%Start Comment
Layout ([ \t\r\f])
ASCII95 ([\040-\176])
AnyQuoted (\\.)
StrChar ([^\"\n\\]|{AnyQuoted})
ChrChar ([^\'\\]|{AnyQuoted})
Idf ([A-Za-z][A-Za-z0-9_\']*)
%%
"||".*$ { /* comment */
}
\"{StrChar}*\" { /* strings */
return_ch('"');
}
\'{ChrChar}\' { /* characters */
return_ch('\'');
}
\%{Layout}*include.* { /* skip %include line */
}
\%{Layout}*insert.* { /* skip %insert line */
}
{Idf} { /* identifier */
return_tk(idf_in_list(yytext, reserved, sizeof reserved, IDF));
}
\n { /* count newlines */
return_eol();
}
{Layout} { /* ignore layout */
}
{ASCII95} { /* copy other text */
return_ch(yytext[0]);
}
. { /* count non-ASCII chars */
lex_non_ascii_cnt++;
}
%%
/* More language-dependent code */
void
yystart(void) {
BEGIN INITIAL;
}

166
core/sim/sim_3_01/newargs.c Normal file
View File

@@ -0,0 +1,166 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: newargs.c,v 2.11 2016-05-13 19:00:52 dick Exp $
*/
#include <stdio.h>
#include "sim.h"
#include "ForEachFile.h"
#include "Malloc.h"
#include "newargs.h"
#define ARGS_INCR 1024
static char *args;
static size_t args_free;
static size_t args_size;
static void
init_args(void) {
args = 0;
args_free = 0;
args_size = 0;
}
static void
add_char_to_args(char ch) {
if (args_free == args_size) {
/* allocated array is full; increase its size */
size_t new_size = args_size + ARGS_INCR;
char *new_args = (char *)Realloc(
(char *)args, sizeof (char *) * new_size
);
args = new_args, args_size = new_size;
}
/* now we are sure there is room enough */
args[args_free++] = ch;
}
static void
add_string_to_args(const Fchar *fn) {
while (*fn) {
add_char_to_args(*fn++);
}
add_char_to_args('\n');
}
static char *
std_input(void) {
/* in the form (name \n)* \0 */
/* get all of standard input */
int ch;
int last_char = '\n';
while (ch = getchar(), ch != EOF) {
/* omit duplicate layout (= empty name) */
if (last_char == '\n' && ch == '\n') continue;
add_char_to_args((char)ch);
last_char = ch;
}
add_char_to_args('\0');
/* make sure the result conforms to the form above */
if (args[args_free-2] != '\n')
fatal("standard input not terminated with newline");
return args;
}
static int
n_names(const char *s) {
int cnt = 0;
while (*s) {
if (*s == '\n') {
cnt++;
}
s++;
}
return cnt;
}
static const char **
new_argv(int argc, char *args) {
/* converts the layout in args to \0, and constructs an argv list */
const char **argv =
(const char **)Malloc((size_t)(argc+1) * sizeof (char *));
char *p = args;
char last_char = '\n';
argc = 0;
while (*p) {
if (last_char == '\n') {
/* here a new name starts */
argv[argc++] = p;
}
last_char = *p;
if (*p == '\n') {
*p = '\0';
}
p++;
}
argv[argc] = 0;
return argv;
}
void
get_new_std_input_args(int *argcp, const char **argvp[]) {
init_args();
char *n_args = std_input();
int argc = n_names(n_args);
const char **argv = new_argv(argc, n_args);
*argcp = argc, *argvp = argv;
}
static int
register_file(const Fchar *fn, const char *msg, const struct stat *fs) {
if (msg) {
fprintf(stderr, "could not handle file %s: %s\n", fn, msg);
return 0;
}
if ( /* it is a non-empty regular file */
S_ISREG(fs->st_mode) && fs->st_size > 0
) {
add_string_to_args(fn);
}
return 1;
}
static char *
recursive_args(int argc, const char *argv[]) {
if (argc == 0) {
ForEachFile(str2Fname("."), register_file);
}
else {
int i;
for (i = 0; i < argc; i++) {
const char *arg = argv[i];
const Fchar *Farg = str2Fname(arg);
if (is_new_old_separator(arg)) {
add_string_to_args(Farg);
} else {
ForEachFile(Farg, register_file);
}
}
}
add_char_to_args('\0');
return args;
}
void
get_new_recursive_args(int *argcp, const char **argvp[]) {
init_args();
char *n_args = recursive_args(*argcp, *argvp);
int argc = n_names(n_args);
const char **argv = new_argv(argc, n_args);
*argcp = argc, *argvp = argv;
}

7
core/sim/sim_3_01/newargs.h Normal file
View File

@@ -0,0 +1,7 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: newargs.h,v 2.4 2012-05-16 07:56:06 dick Exp $
*/
extern void get_new_std_input_args(int *argcp, char const **argvp[]);
extern void get_new_recursive_args(int *argcp, const char **argvp[]);

5
core/sim/sim_3_01/option-i.inp Normal file
View File

@@ -0,0 +1,5 @@
pass1.c
pass2.c
/
pass3.c
../teckel

214
core/sim/sim_3_01/options.c Normal file
View File

@@ -0,0 +1,214 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: options.c,v 1.20 2017-03-19 09:30:37 dick Exp $
*/
#include <stdio.h>
#include <stdlib.h>
#include "sim.h"
#include "token.h"
#include "language.h"
#include "options.h"
static char options[128];
static void bad_option_exit(
const char *progname, const struct option *optlist,
char *msg, int c
);
static int opt_value(
const char *progname, const struct option *op,
const char *arg, const char *argv[]
);
static int do_arg(
const char *progname, const struct option *optlist,
const char *arg, const char *argv[]
);
int
do_options(
const char *progname, const struct option *optlist,
int argc, const char *argv[]
) {
int skips = 0;
while (argc > 0 && argv[0][0] == '-') {
int consumed = do_arg(progname, optlist, &argv[0][1], argv);
argc -= consumed, argv += consumed, skips += consumed;
}
return skips;
}
void
set_option(char ch) {
options[(int)ch]++;
}
int
is_set_option(int ch) {
return options[ch];
}
static int
do_arg(
const char *progname, const struct option *optlist,
const char *arg, const char *argv[]
) {
int consumed = 0;
while (*arg) {
/* treat argument character */
char opc = *arg++;
const struct option *op;
for (op = optlist; op->op_char; op++) {
if (opc == op->op_char) {
set_option(opc);
if (op->op_type != None) {
consumed = opt_value(
progname, op, arg, argv
);
}
break;
}
}
if (!op->op_char) {
bad_option_exit(progname, optlist,
"option -%c unknown", opc
);
/*NOTREACHED*/
}
if (consumed) break;
}
if (!consumed) {
consumed = 1;
}
return consumed;
}
static int
opt_value(
const char *progname, const struct option *op,
const char *arg, const char *argv[]
) {
/* get the string and the number of args consumed */
const char *string;
int consumed;
if (*arg) {
string = arg, consumed = 1;
}
else if (argv[1]) {
string = argv[1], consumed = 2;
} else {
string = 0, consumed = 0;
}
if (!string || !*string) {
bad_option_exit(progname, (struct option *)0,
"option -%c requires another argument", op->op_char
);
/*NOTREACHED*/
}
switch (op->op_type) {
case Number:
*(int *)op->op_value = atoi(string);
break;
case String:
*(const char **)op->op_value = string;
break;
}
return consumed;
}
void
allow_at_most_one_option_out_of(const char *opts) {
const char *first;
for (first = opts; *first; first++) {
const char *second;
for (second = first + 1; *second; second++) {
if (is_set_option(*first) &&is_set_option(*second)) {
char msg[256];
sprintf(msg,
"options -%c and -%c are incompatible",
*first, *second
);
fatal(msg);
}
}
}
}
static void
bad_option_exit(
const char *progname, const struct option *optlist, char *msg, int c
) {
fprintf(stderr, "%s: ", progname);
fprintf(stderr, msg, c);
fprintf(stderr, "\n");
fprintf(stderr, "Possible options are:\n");
const struct option *op;
for (op = optlist; op->op_char; op++) {
if (op->op_char == ' ') {
fprintf(stderr, "\n\t\t%s\n", op->op_text);
} else {
fprintf(stderr, "\t-%c%c\t%s\n",
op->op_char,
( op->op_type == Number ? 'N' :
op->op_type == String ? 'F' :
' '
),
op->op_text
);
}
}
exit(1);
}
static int
is_essential_option(char op_char) {
if (op_char == 'r') return 1;
if (op_char == 'w') return 1;
if (is_set_option('p') && op_char == 't') return 1;
return 0;
}
void
print_options(const char *progname, const struct option *optlist) {
const struct option *op;
if (!is_set_option('T')) {
fprintf(stdout, "%s (%s)\n", progname, Version);
fprintf(stdout, "Subject: %s\n", Subject);
}
fprintf(stdout, "Option settings:");
if (!is_set_option('T')) fprintf(stdout, "\n");
for (op = optlist; op->op_char; op++) {
if ( is_set_option(op->op_char)
|| is_essential_option(op->op_char)
) {
fprintf(stdout, " -%c", op->op_char);
switch (op->op_type) {
case None: break;
case Number:
fprintf(stdout, "%d",
*(int *)op->op_value);
break;
case String:
fprintf(stdout, " %s",
*(const char **)op->op_value);
break;
}
if (!is_set_option('T')) {
fprintf(stdout, " (%s)\n", op->op_text);
}
}
}
if (is_set_option('T')) fprintf(stdout, "\n");
}

24
core/sim/sim_3_01/options.h Normal file
View File

@@ -0,0 +1,24 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: options.h,v 1.12 2016-06-12 13:00:04 dick Exp $
*/
/* Setting and consulting command line options
*/
enum Value_Type {None, Number, String};
struct option {
char op_char; /* char as in call */
char *op_text; /* explanatory text */
enum Value_Type op_type;
void *op_value;
};
extern void set_option(char ch);
extern int is_set_option(int ch);
extern int do_options(
const char *progname, const struct option *optlist,
int argc, const char *argv[]
);
extern void allow_at_most_one_option_out_of(const char *opts);
extern void print_options(const char *progname, const struct option *optlist);

264
core/sim/sim_3_01/pascallang.l Normal file
View File

@@ -0,0 +1,264 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: pascallang.l,v 2.22 2017-03-19 09:23:20 dick Exp $
*/
/*
PASCAL language front end for the similarity tester.
Author: Maarten van der Meulen <maarten@cs.vu.nl>
Date: May 1986
*/
#include "options.h"
#include "token.h"
#include "language.h"
#include "algollike.h"
#include "idf.h"
#include "lex.h"
#include "lang.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent data */
/* Data for module idf */
static const struct idf ppcmd[] = {
{"define", META('d')},
{"else", META('e')},
{"endif", META('E')},
{"if", META('i')},
{"ifdef", META('I')},
{"ifndef", META('x')},
{"include", MTCT('I')},
{"line", META('l')},
{"undef", META('u')}
};
static const struct idf reserved[] = {
{"and", NORM('&')},
{"array", NORM('A')},
{"begin", NORM('{')},
{"case", NORM('c')},
{"const", NORM('C')},
{"div", NORM('/')},
{"do", NORM('D')},
{"downto", NORM('d')},
{"else", NORM('e')},
{"end", NORM('}')},
{"extern", CTRL('E')},
{"file", NORM('F')},
{"for", NORM('f')},
{"function", NORM('p')}, /* Equal to procedure */
{"goto", NORM('g')},
{"if", NORM('i')},
{"in", NORM('I')},
{"label", NORM('l')},
{"mod", NORM('%')},
{"nil", NORM('n')},
{"not", NORM('!')},
{"of", No_Token},
{"or", NORM('|')},
{"packed", NORM('P')},
{"procedure", NORM('p')},
{"program", No_Token},
{"record", NORM('r')},
{"repeat", NORM('R')},
{"set", NORM('s')},
{"then", No_Token},
{"to", NORM('t')},
{"type", NORM('T')},
{"until", NORM('u')},
{"var", NORM('v')},
{"while", NORM('w')},
{"with", NORM('W')}
};
/* Special treatment of identifiers */
static Token
idf2token(int hashing) {
Token tk;
lower_case(yytext); /* Pascal is case-insensitive */
tk = idf_in_list(yytext, reserved, sizeof reserved, IDF);
if (Token_EQ(tk, IDF) && hashing) {
/* return a one-Token hash code */
tk = idf_hashed(yytext);
}
return tk;
}
/* Token sets for module algollike */
const Token Non_Finals[] = {
IDF, /* identifier */
NORM('{'), /* also begin */
NORM('('),
NORM('['),
NORM('A'), /* array */
NORM('c'), /* case */
NORM('C'), /* const */
NORM('/'), /* div */
CTRL('E'), /* extern */
NORM('F'), /* file */
NORM('f'), /* for */
NORM('g'), /* goto */
NORM('i'), /* if */
NORM('l'), /* label */
NORM('P'), /* packed */
NORM('p'), /* procedure/function */
NORM('r'), /* record */
NORM('R'), /* repeat */
NORM('s'), /* set */
NORM('T'), /* type */
NORM('v'), /* var */
NORM('w'), /* while */
NORM('W'), /* with */
No_Token
};
const Token Non_Initials[] = {
NORM(')'),
NORM('}'),
NORM(';'),
No_Token
};
const Token Openers[] = {
NORM('{'),
NORM('('),
NORM('['),
No_Token
};
const Token Closers[] = {
NORM('}'),
NORM(')'),
NORM(']'),
No_Token
};
/* Language-dependent code */
const char *Subject = "Pascal programs";
void
Init_Language(void) {
Init_Algol_Language(Non_Finals, Non_Initials, Openers, Closers);
}
int
May_Be_Start_Of_Run(Token ch) {
return May_Be_Start_Of_Algol_Run(ch);
}
size_t
Best_Run_Size(const Token *str, size_t size) {
return Best_Algol_Run_Size(str, size);
}
%}
%option noyywrap
%Start Comment
Layout ([ \t\r\f])
ASCII95 ([\040-\176])
AnyQuoted (\\.)
StrChar ([^\'\n\\]|{AnyQuoted})
StartComment ("{"|"(*")
EndComment ("}"|"*)")
SafeComChar ([^*}\n])
UnsafeComChar ("*")
Digit ([0-9])
Idf ([A-Za-z][A-Za-z0-9_]*)
%%
{StartComment} { /* See clang.l */
BEGIN Comment;
}
<Comment>{SafeComChar}+ { /* safe comment chunk */
}
<Comment>{UnsafeComChar} { /* unsafe char, read one by one */
}
<Comment>"\n" { /* to break up long comments */
return_eol();
}
<Comment>{EndComment} { /* end-of-comment */
BEGIN INITIAL;
}
\'{StrChar}*\' { /* character strings */
return_ch('"');
}
^#{Layout}*include.* { /* ignore #include lines */
}
^#{Layout}*{Idf} { /* a preprocessor line */
char *idf = yytext+1;
/* skip layout in front of preprocessor identifier */
while (*idf == ' ' || *idf == '\t') {
idf++;
}
return_tk(idf_in_list(idf, ppcmd, sizeof ppcmd, NORM('#')));
}
{Digit}+ { /* numeral, passed as an identifier */
return_tk(IDF);
}
{Idf}/"(" { /* identifier in front of ( */
Token tk;
tk = idf2token(is_set_option('F'));
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
{Idf} { /* identifier */
Token tk;
tk = idf2token(0 /* no hashing */);
if (!Token_EQ(tk, No_Token)) return_tk(tk);
}
\; { /* semicolon, conditionally ignored */
if (is_set_option('f')) return_ch(yytext[0]);
}
\n { /* count newlines */
return_eol();
}
{Layout} { /* ignore layout */
}
{ASCII95} { /* copy other text */
return_ch(yytext[0]);
}
. { /* count non-ASCII chars */
lex_non_ascii_cnt++;
}
%%
/* More language-dependent code */
void
yystart(void) {
BEGIN INITIAL;
}

177
core/sim/sim_3_01/pass1.c Normal file
View File

@@ -0,0 +1,177 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: pass1.c,v 2.35 2016-07-29 12:50:02 dick Exp $
*/
#include <stdio.h>
#include <string.h>
#include "debug.par"
#include "sim.h"
#include "text.h"
#include "token.h"
#include "tokenarray.h"
#include "lang.h"
#include "options.h"
#include "pass1.h"
#ifdef DB_TEXT
static void db_print_text(const struct text *);
#endif
static void fprint_count(FILE *f, size_t cnt, const char *);
void
Read_Input_Files(int argc, const char *argv[]) {
int n;
Init_Text(argc);
Init_Token_Array();
/* Initially assume all texts to be new */
Number_of_New_Texts = Number_of_Texts;
/* Read the files */
for (n = 0; n < Number_of_Texts; n++) {
const char *fname = argv[n];
struct text *txt = &Text[n];
if (!is_set_option('T')) {
fprintf(Output_File, "File %s: ", fname);
}
txt->tx_fname = fname;
txt->tx_pos = 0;
txt->tx_start = Token_Array_Length();
txt->tx_limit = Token_Array_Length();
if (is_new_old_separator(fname)) {
if (!is_set_option('T')) {
fprintf(Output_File, "new/old separator\n");
}
if (Number_of_New_Texts == Number_of_Texts) {
Number_of_New_Texts = n;
} else fatal("more than one new/old separator");
}
else {
int file_opened = 0;
if (Open_Text(First_Pass, txt)) {
file_opened = 1;
} else {
/* print a warning */
if (is_set_option('T')) {
/* the file name has not yet been
printed; print it now
*/
fprintf(Output_File, "File %s: ",
fname);
}
fprintf(Output_File,
">>>> cannot open <<<<\n");
/* the file has still been opened
with a null file for uniformity
*/
}
while (Next_Text_Token_Obtained()) {
if (!Token_EQ(lex_token, End_Of_Line)) {
Store_Token(lex_token);
}
}
Close_Text(First_Pass, txt);
txt->tx_limit = Token_Array_Length();
txt->tx_EOL_terminated =
Token_EQ(lex_token, End_Of_Line);
/* report */
if (file_opened && !is_set_option('T')) {
fprint_count(Output_File,
txt->tx_limit - txt->tx_start,
Token_Name
);
fprintf(Output_File, ", ");
fprint_count(Output_File,
lex_nl_cnt - 1 +
(!txt->tx_EOL_terminated ? 1 : 0),
"line"
);
if (!txt->tx_EOL_terminated) {
fprintf(Output_File,
" (not NL-terminated)");
}
if (lex_non_ascii_cnt) {
fprintf(Output_File, ", ");
fprint_count(Output_File,
lex_non_ascii_cnt,
"non-ASCII character"
);
}
fprintf(Output_File, "\n");
}
#ifdef DB_TEXT
db_print_text(txt);
#endif /* DB_TEXT */
}
fflush(Output_File);
}
/* report total */
int sep_present = (Number_of_Texts != Number_of_New_Texts);
fprintf(Output_File, "Total input: ");
fprint_count(Output_File,
(!sep_present ? Number_of_Texts : Number_of_Texts - 1),
"file"
);
fprintf(Output_File, " (%d new, %d old), ",
Number_of_New_Texts,
(!sep_present ? 0 : Number_of_Texts - Number_of_New_Texts - 1)
);
fprint_count(Output_File, Token_Array_Length() - 1, Token_Name);
fprintf(Output_File, "\n\n");
fflush(Output_File);
}
static void
fprint_count(FILE *f, size_t cnt, const char *unit) {
/* Prints a grammatically correct string "%u %s[s]"
for units that form their plural by suffixing -s.
*/
fprintf(f, "%s %s%s", size_t2string(cnt), unit, (cnt == 1 ? "" : "s"));
}
#ifdef DB_TEXT
static void
db_print_text(const struct text *txt) {
/* prints a text (in compressed form) */
size_t i;
fprintf(Debug_File, "\n\n**** DB_PRINT_TEXT ****\n");
fprintf(Debug_File, "File \"%s\", %s %ss, ",
txt->tx_fname,
size_t2string(txt->tx_limit - txt->tx_start),
Token_Name
);
fprintf(Debug_File, "txt->tx_start = %s, txt->tx_limit = %s\n",
size_t2string(txt->tx_start),
size_t2string(txt->tx_limit)
);
int BoL = 1;
for (i = txt->tx_start; i < txt->tx_limit; i++) {
if (BoL) {
fprintf(Debug_File, "[%s]:", size_t2string(i));
BoL = 0;
}
fprintf(Debug_File, " ");
fprint_token(Debug_File, Token_Array[i]);
if ((i - txt->tx_start + 1) % 10 == 0) {
fprintf(Debug_File, "\n");
BoL = 1;
}
}
fprintf(Debug_File, "\n");
}
#endif /* DB_TEXT */

9
core/sim/sim_3_01/pass1.h Normal file
View File

@@ -0,0 +1,9 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: pass1.h,v 1.8 2015-05-04 19:49:11 dick Exp $
*/
/* Reads the input files; stores the tokens in Token Token_Array[]
and the input file descriptions in struct text text[].
*/
extern void Read_Input_Files(int argc, const char *argv[]);

170
core/sim/sim_3_01/pass2.c Normal file
View File

@@ -0,0 +1,170 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: pass2.c,v 2.24 2016-05-13 19:00:53 dick Exp $
*/
#include <stdio.h>
#include "debug.par"
#include "sim.h"
#include "token.h"
#include "text.h"
#include "lang.h"
#include "pass2.h"
#ifdef DB_POS
static void db_print_pos_list(const char *, const struct text *);
static void db_print_lex(const char *);
#endif
static void pass2_txt(struct text *txt);
void
Retrieve_Runs(void) {
int n;
for (n = 0; n < Number_of_Texts; n++) {
pass2_txt(&Text[n]);
}
}
/* begin instantiate static void sort_pos_list(struct position **) */
#define SORT_STRUCT position
#define SORT_NAME sort_pos_list
#define SORT_BEFORE(p1,p2) ((p1)->ps_tk_cnt < (p2)->ps_tk_cnt)
#define SORT_NEXT ps_next
#include "sortlist.bdy"
/* end instantiate sort_pos_list() */
static void
pass2_txt(struct text *txt) {
struct position *pos;
size_t old_nl_cnt;
if (!txt->tx_pos) /* no need to scan the file */
return;
/* Open_Text() initializes lex_nl_cnt and lex_tk_cnt */
if (!Open_Text(Second_Pass, txt)) {
fprintf(stderr, ">>>> File %s disappeared <<<<\n",
txt->tx_fname
);
return;
}
/* Sort the positions so they can be matched to the file; the linked
list of struct positions snakes through the struct positions in the
struct chunks in the struct runs.
*/
#ifdef DB_POS
db_print_pos_list("before sorting", txt);
#endif /* DB_POS */
sort_pos_list(&txt->tx_pos);
#ifdef DB_POS
db_print_pos_list("after sorting", txt);
#endif /* DB_POS */
#ifdef DB_NL_BUFF
db_print_nl_buff(txt->tx_nl_start, txt->tx_nl_limit);
#endif /* DB_NL_BUFF */
#ifdef DB_POS
fprintf(Debug_File, "\n**** DB_PRINT_SCAN of %s ****\n", txt->tx_fname);
#endif /* DB_POS */
old_nl_cnt = 1;
pos = txt->tx_pos;
while (pos) {
/* we scan the pos list and the file in parallel */
/* find the corresponding line */
while (pos->ps_tk_cnt >= lex_tk_cnt) {
/* pos does not refer to this line, try the next */
/* shift the administration */
old_nl_cnt = lex_nl_cnt;
/* and get the next eol position */
if (!Next_Text_EOL_Obtained()) {
/* reached end of file without obtaining EOL */
if (!txt->tx_EOL_terminated) {
/* that's OK then */
} else {
fprintf(stderr,
">>>> File %s modified <<<<\n",
txt->tx_fname
);
}
break;
}
#ifdef DB_POS
db_print_lex(txt->tx_fname);
#endif /* DB_POS */
}
/* fill in the pos */
switch (pos->ps_type) {
case 0: /* first token of run */
pos->ps_nl_cnt = old_nl_cnt;
break;
case 1: /* last token of run */
pos->ps_nl_cnt = lex_nl_cnt;
break;
}
/* and get the next pos */
pos = pos->ps_next;
}
#ifdef DB_POS
db_print_pos_list("after scanning", txt);
#endif /* DB_POS */
/* Flush the flex buffers; it's easier than using YY_BUFFER_STATE. */
while (Next_Text_EOL_Obtained());
Close_Text(Second_Pass, txt);
}
#ifdef DB_POS
static void
db_print_pos(const struct position *pos) {
fprintf(Debug_File, "pos type = %s; %s count = %u",
(pos->ps_type == 0 ? "first" : " last"),
Token_Name,
pos->ps_tk_cnt
);
fprintf(Debug_File, ", line # = ");
if (pos->ps_nl_cnt == (size_t) -1) {
fprintf(Debug_File, "<NOT SET>");
}
else {
fprintf(Debug_File, "%u", pos->ps_nl_cnt);
}
fprintf(Debug_File, "\n");
}
static void
db_print_pos_list(const char *msg, const struct text *txt) {
fprintf(Debug_File, "\n**** DB_PRINT_POS_LIST of %s, %s ****\n",
txt->tx_fname, msg);
const struct position *pos = txt->tx_pos;
while (pos) {
db_print_pos(pos);
pos = pos->ps_next;
}
fprintf(Debug_File, "\n");
}
static void
db_print_lex(const char *fn) {
fprintf(Debug_File,
"%s: lex_tk_cnt = %u, lex_nl_cnt = %u, lex_token = ",
fn, lex_tk_cnt, lex_nl_cnt);
fprint_token(Debug_File, lex_token);
fprintf(Debug_File, "\n");
}
#endif /* DB_POS */

9
core/sim/sim_3_01/pass2.h Normal file
View File

@@ -0,0 +1,9 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: pass2.h,v 1.3 2012-06-05 09:58:53 dick Exp $
*/
/* Determines for each position that is part of a run, at which
line number it starts and ends.
*/
extern void Retrieve_Runs(void);

351
core/sim/sim_3_01/pass3.c Normal file
View File

@@ -0,0 +1,351 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: pass3.c,v 2.31 2016-07-31 18:55:44 dick Exp $
*/
#include <stdio.h>
#include <string.h>
#include "system.par"
#include "settings.par"
#include "debug.par"
#include "sim.h"
#include "text.h"
#include "token.h"
#include "runs.h"
#include "options.h"
#include "pass3.h"
#include "percentages.h"
#ifdef DB_RUN
#include "tokenarray.h"
static void db_run(const struct run *);
#endif
/* Positioning of UTF-8 characters must be done with a finer grain than just
10 Courier characters to the inch. On the other hand we do not know the
exact width of each UNICODE character, so whatever we do is an approximation.
We use a granularity of 1 pt and a font size of 10 pts.
Since C does not have type-checked integer subtypes, we put the the fact
that they handle UTF-8 chars in the variable and routine names to avoid
errors.
*/
typedef int pts;
#define FONT_SIZE (10)
static FILE *open_chunk(const struct chunk *);
static void print_char(char);
static void print_spaces(int);
static pts print_UTF8_line(FILE *);
static pts print_UTF8_char(int ch, FILE *);
static void print_UTF8_spaces(pts);
static void show_run(const struct run *);
static void show_1C_line(FILE *, const char *);
static int print_header(const struct chunk *);
static int print_string(const char *);
static int print_size_t(size_t);
static int length_size_t(size_t);
static int max_line_length; /* Actual maximum line length */
static pts max_line_length_UTF8;
void
Show_Runs(void) {
#ifdef DB_RUN
fprintf(Debug_File, "Starting Show_Runs()\n");
#endif /* DB_RUN */
const struct run *run = /*ZZ*/
(is_set_option('u') ? unsorted_runs() : sorted_runs());
while (run) {
#ifdef DB_RUN
db_run(run);
#endif /* DB_RUN */
show_run(run);
print_char('\n');
fflush(Output_File);
run = run->rn_next;
}
discard_runs();
}
static void
show_run(const struct run *run) {
max_line_length = Page_Width / 2 - 1;
max_line_length_UTF8 = max_line_length * FONT_SIZE;
/* The animals came in two by two ... */
const struct chunk *cnk0 = &run->rn_chunk0;
const struct chunk *cnk1 = &run->rn_chunk1;
size_t nl_cnt0 = cnk0->ch_last.ps_nl_cnt - cnk0->ch_first.ps_nl_cnt;
size_t nl_cnt1 = cnk1->ch_last.ps_nl_cnt - cnk1->ch_first.ps_nl_cnt;
FILE *f0;
FILE *f1;
/* display heading of chunk */
if (!is_set_option('d')) {
/* no assumptions about the lengths of the file names! */
size_t size = run->rn_size;
int pos = print_header(cnk0);
print_spaces(max_line_length - pos);
print_char('|');
pos = print_header(cnk1);
print_spaces(max_line_length - pos - length_size_t(size) - 2);
fprintf(Output_File, "[%s]\n", size_t2string(size));
}
else {
/* diff-like format */
(void)print_header(cnk0);
print_char('\n');
(void)print_header(cnk1);
print_char('\n');
}
/* stop if that suffices */
if (is_set_option('n'))
return; /* ... had enough so soon ... */
/* open the files that hold the chunks */
f0 = open_chunk(cnk0);
f1 = open_chunk(cnk1);
/* display the chunks in the required format */
if (!is_set_option('d')) {
/* print 2-column format */
while (nl_cnt0 != 0 || nl_cnt1 != 0) {
int pos_UTF8 = 0;
if (nl_cnt0) {
pos_UTF8 = print_UTF8_line(f0);
nl_cnt0--;
}
print_UTF8_spaces(max_line_length_UTF8 - pos_UTF8);
print_char('|');
if (nl_cnt1) {
(void)print_UTF8_line(f1);
nl_cnt1--;
}
print_char('\n');
}
}
else {
/* display the chunks in a diff(1)-like format */
while (nl_cnt0--) {
show_1C_line(f0, "<");
}
(void)print_string("---\n");
while (nl_cnt1--) {
show_1C_line(f1, ">");
}
}
/* close the pertinent files */
fclose(f0);
fclose(f1);
}
static int
print_header(const struct chunk *cnk) {
int pos = 0;
pos += print_string(cnk->ch_text->tx_fname);
pos += print_string(": line ");
pos += print_size_t(cnk->ch_first.ps_nl_cnt);
pos += print_string("-");
pos += print_size_t(cnk->ch_last.ps_nl_cnt - 1);
return pos;
}
static int
print_string(const char *str) {
fprintf(Output_File, "%s", str);
return (int) strlen(str);
}
static int
print_size_t(size_t u) {
fprintf(Output_File, "%s", size_t2string(u));
return length_size_t(u);
}
static int
length_size_t(size_t u) {
int res = 1;
while (u > 9) {
u /= 10, res++;
}
return res;
}
static FILE *
open_chunk(const struct chunk *cnk) {
/* Opens the file in which the chunk resides, positions the
file at the beginning of the chunk and returns the file pointer.
Note that we use fopen() here, which opens a character stream,
rather than Open_Text(), which opens a token stream.
*/
const char *fname = cnk->ch_text->tx_fname;
FILE *f = fopen(fname, "r");
size_t nl_cnt;
if (!f) {
fprintf(stderr, ">>>> File %s disappeared <<<<\n", fname);
f = fopen(NULLFILE, "r");
}
nl_cnt = cnk->ch_first.ps_nl_cnt;
while (nl_cnt > 1) {
int ch = getc(f);
if (ch < 0) break;
if (ch == '\n') {
nl_cnt--;
}
}
return f;
}
static pts
print_UTF8_line(FILE *f) {
/* Reads one line from f and prints it in condensed form. */
int indent = 0, pos_UTF8 = 0;
int ch;
/* condense initial blanks */
while ((ch = getc(f)), ch == ' ' || ch == '\t') {
if (ch == '\t') {
indent = 8;
}
else {
indent++;
}
if (indent == 8) {
/* every eight blanks give one blank */
if (pos_UTF8 < max_line_length_UTF8) {
pos_UTF8 += print_UTF8_char(' ', 0);
}
indent = 0;
}
}
/* print the rest */
while (ch >= 0 && ch != '\n') {
if (ch == '\t') {
/* replace tabs by blanks */
ch = ' ';
}
if (pos_UTF8 < max_line_length_UTF8) {
pos_UTF8 += print_UTF8_char(ch, f);
}
ch = getc(f);
}
return pos_UTF8;
}
static void
print_char(char ch) {
fprintf(Output_File, "%c", ch);
}
static void
print_spaces(int n) {
while (n > 0) {
print_char(' '), --n;
}
}
static pts
print_UTF8_char(int ch, FILE *f) {
int pos_UTF8 = FONT_SIZE;
fprintf(Output_File, "%c", ch);
if (ch < 192) return pos_UTF8;
while (ch & 0x40) {
int ch1 = getc(f);
fprintf(Output_File, "%c", ch1);
if ((ch1 & 0xC0) != 0x80) {
/* bad UTF-8 */
return pos_UTF8; /* no recovery */
}
pos_UTF8 += 4;
/* ^ Stupid heuristic: the longer the UTF sequence, the bigger
the character. Rough, but the best we can reasonably do.
*/
ch <<= 1;
}
return pos_UTF8;
}
static void
print_UTF8_spaces(pts n) {
while (n > 0) {
print_char(' '), n -= FONT_SIZE;
}
}
static void
show_1C_line(FILE *f, const char *marker) {
/* displays one line from f, preceded by the marker
*/
int ch;
fprintf(Output_File, "%s ", marker);
while ((ch = getc(f)), ch > 0 && ch != '\n') {
print_char(ch);
}
print_char('\n');
}
#ifdef DB_RUN
static void db_chunk(const struct chunk *);
static void
db_run(const struct run *run) {
/* prints detailed data about a run */
const struct chunk *cnk0 = &run->rn_chunk0;
const struct chunk *cnk1 = &run->rn_chunk1;
db_run_info(0, run, 1);
db_chunk(cnk0);
db_chunk(cnk1);
}
static void
db_chunk(const struct chunk *cnk) {
/* print the tokens in the chunk, with a one-char margin
*/
size_t i;
const struct position *first = &cnk->ch_first;
const struct position *last = &cnk->ch_last;
size_t start = cnk->ch_text->tx_start;
if (first->ps_tk_cnt > 0) {
fprintf(Debug_File, "...");
fprint_token(Debug_File,
Token_Array[start + first->ps_tk_cnt - 1]);
fprintf(Debug_File, " ");
}
else { /* create same offset as above */
fprintf(Debug_File, " ");
}
for (i = first->ps_tk_cnt; i <= last->ps_tk_cnt; i++) {
fprintf(Debug_File, " ");
fprint_token(Debug_File, Token_Array[start + i]);
}
if (start + last->ps_tk_cnt + 1 < cnk->ch_text->tx_limit) {
fprintf(Debug_File, " ");
fprint_token(Debug_File,
Token_Array[start + last->ps_tk_cnt + 1]);
fprintf(Debug_File, "...");
}
fprintf(Debug_File, "\n");
}
#endif /* DB_RUN */

7
core/sim/sim_3_01/pass3.h Normal file
View File

@@ -0,0 +1,7 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: pass3.h,v 1.3 2012-06-05 09:58:53 dick Exp $
*/
/* Print the contents of runs */
extern void Show_Runs(void);

263
core/sim/sim_3_01/percentages.c Normal file
View File

@@ -0,0 +1,263 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: percentages.c,v 1.31 2016-07-31 18:55:44 dick Exp $
*/
#include <stdio.h>
#include "debug.par"
#include "sim.h"
#include "text.h"
#include "options.h"
#include "Malloc.h"
#include "percentages.h"
struct match {
struct match *ma_next;
const char *ma_fname0;
const char *ma_fname1;
size_t ma_size; /* # tokens of file 0 found in file 1 */
size_t ma_size0; /* # tokens in file 0 */
};
static struct match *match_list = 0; /* to be allocated by new() */
static void
do_add_to_percentages(
int rec_level, const struct text *txt0, const struct text *txt1,
size_t size);
static void print_perc_info(const struct match *m);
#ifdef DB_PERC
static void db_print_match(const struct match *ma);
static void db_print_match_list(const char *msg);
#endif
void
add_to_percentages(
const struct text *txt0, const struct text *txt1, size_t size
) {
/* percentages are only meaningful between different files */
if (txt0 == txt1) return;
do_add_to_percentages(0, txt0, txt1, size);
}
/* The important point to notice is that when a certain combination of two
texts turns up as a run, all runs with that combination will follow
immediately, and once a different combination has appeared the first one
will never return.
*/
static void
print_and_remove_match_list(void) {
while (match_list) {
struct match *m = match_list;
match_list = m->ma_next;
print_perc_info(m);
Free(m);
}
}
static void
do_add_to_percentages(
int rec_level, const struct text *txt0, const struct text *txt1, size_t size
) {
#ifdef DB_PERC
fprintf(Debug_File, "do_add: %s, %s, %d\n",
txt0->tx_fname,
txt1->tx_fname,
size);
#endif
/* look up the (txt0, txt1) combination in the match list */
struct match *m = match_list;
while (m) {
if ( m->ma_fname0 == txt0->tx_fname
&& m->ma_fname1 == txt1->tx_fname
) {
/* found it; now update it */
m->ma_size += size;
#ifdef DB_PERC
fprintf(Debug_File, "match updated:\n");
db_print_match(m);
#endif
return;
}
m = m->ma_next;
}
{ /* it's not there; create a new entry, but tread carefully */
m = try_new(struct match);
if (m == 0) {
/* Normally this should not happen. Freeing the
forward_reference[] and the last_index[] tables
should free ample memory to store the administration
of the runs found. But the tables are linear in the
input length and the number of runs is more or less
quadratic, so for massive comparisons the runs may
overtake the tables.
We then free memory by flushing collected results,
sacrificing the sorting.
*/
/* try to recover */
if (rec_level > 0) {
/* but only one chance, to prevent looping */
OutOfMemoryExit(0);
/*NOTREACHED*/
}
static int message_given;
if (!message_given) {
fprintf(Output_File,
">>>> OUTPUT NOT SORTED <<<<\n");
message_given = 1;
}
print_and_remove_match_list();
/* and try again */
do_add_to_percentages(1, txt0, txt1, size);
return;
}
m->ma_next = match_list;
match_list = m;
m->ma_fname0 = txt0->tx_fname;
m->ma_fname1 = txt1->tx_fname;
m->ma_size = size;
m->ma_size0 = txt0->tx_limit - txt0->tx_start;
#ifdef DB_PERC
fprintf(Debug_File, "match created:\n");
db_print_match(m);
#endif /* DB_PERC */
}
if (is_set_option('u')) {
/* Print the file/file combination as soon as it is finished.
It can only be found to be finished when the next pair is
found. Then a new struct match is created, on top of the
old one. So we check for the existence of a second match
and then print and release it.
We agree that this is a trick.
*/
if (match_list->ma_next) {
print_perc_info(match_list->ma_next);
Free(match_list->ma_next);
match_list->ma_next = 0;
}
}
}
/* PRINTING */
/* We want the sorting order
all contributors of the file with the highest percentage
all contributors of the file with the next lower percentage
etc.
but this order cannot be specified by a single SORT_BEFORE().
So we sort for percentage, and then reorder during printing.
*/
/* instantiate sort_match_list(struct match **listhook) */
static float
match_percentage(const struct match *m) {
return (((float)m->ma_size)/((float)m->ma_size0));
}
#define SORT_STRUCT match
#define SORT_NAME sort_match_list
#define SORT_BEFORE(p1,p2) (match_percentage(p1) > match_percentage(p2))
#define SORT_NEXT ma_next
#include "sortlist.bdy"
#ifdef DB_PERC
static void
db_print_match(const struct match *ma) {
fprintf(Debug_File, "%s < %s, %d/%d=%3.2f%%\n",
ma->ma_fname0, ma->ma_fname1,
ma->ma_size, ma->ma_size0,
match_percentage(ma)*100.0
);
}
static void
db_print_match_list(const char *msg) {
fprintf(Debug_File, "\n\n**** DB_PERCENTAGES %s ****\n", msg);
const struct match *ma;
for (ma = match_list; ma; ma = ma->ma_next) {
db_print_match(ma);
}
fprintf(Debug_File, "\n");
}
#endif /* DB_PERC */
static void
print_perc_info(const struct match *m) {
int mp = (int)(match_percentage(m)*100.0 + 0.5 /* rounding */);
if (mp >= Threshold_Percentage) {
fprintf(Output_File,
"%s consists for %d %% of %s material\n",
m->ma_fname0, mp, m->ma_fname1
);
fflush(Output_File);
}
}
static void
print_and_remove_perc_info_for_top_file(struct match **m_hook) {
struct match *m = *m_hook;
const char *fname = m->ma_fname0;
print_perc_info(m); /* always print main contributor */
*m_hook = m->ma_next;
Free(m);
/* This is a horrible piece of code that should be rewritten.
It works only because initially m_hook points to match_list,
so removing entries with the same file name updates match_list,
whereas as soon as another file name is met, m_hook starts pointing
to the next field in the previous link,
so removing entries with the same file name just removes the entry.
Sneaky.
*/
while ((m = *m_hook)) {
if (m->ma_fname0 == fname) {
/* print subsequent contributors only if not
suppressed by -P
*/
if (!is_set_option('P')) {
print_perc_info(m);
}
/* remove the struct */
*m_hook = m->ma_next;
Free(m);
} else {
/* skip the struct */
m_hook = &m->ma_next;
}
}
}
static void
print_percentages(void) {
#ifdef DB_PERC
db_print_match_list("before listing match_list");
#endif /* DB_PERC */
/* destroys the match list while printing */
while (match_list) {
print_and_remove_perc_info_for_top_file(&match_list);
}
}
void
Show_Percentages(void) {
#ifdef DB_PERC
db_print_match_list("before sort");
#endif /* DB_PERC */
sort_match_list(&match_list);
#ifdef DB_PERC
db_print_match_list("after sort");
#endif /* DB_PERC */
print_percentages();
}

8
core/sim/sim_3_01/percentages.h Normal file
View File

@@ -0,0 +1,8 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: percentages.h,v 1.6 2016-07-31 18:55:44 dick Exp $
*/
extern void add_to_percentages(
const struct text *txt0, const struct text *txt1, size_t size);
extern void Show_Percentages(void);

144
core/sim/sim_3_01/runs.c Normal file
View File

@@ -0,0 +1,144 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: runs.c,v 1.17 2016-07-31 18:55:44 dick Exp $
*/
#include "sim.h"
#include "text.h"
#include "runs.h"
#include "Malloc.h"
#include "debug.par"
static struct run *runs;
static void set_chunk(
struct chunk *cnk, struct text *txt, size_t start, size_t size);
static void set_pos(
struct position *pos, int type, struct text *txt, size_t start);
void
add_to_runs(
struct text *txt0, size_t i0, struct text *txt1, size_t i1,
size_t size
) {
struct run *r = new(struct run);
/* This should always succeed, since releasing the forward_reference[]
and the last_index[] tables will have freed large amounts of memory,
and massive comparisons as in percentage.c are not reasonable and
quite useless here, due to the enormous amount of output to be
expected.
*/
set_chunk(&r->rn_chunk0, txt0, i0 - txt0->tx_start, size);
set_chunk(&r->rn_chunk1, txt1, i1 - txt1->tx_start, size);
r->rn_size = size;
#ifdef DB_RUN
fprintf(Debug_File, "add_to_runs: %s, %s, %d\n",
r->rn_chunk0.ch_text->tx_fname,
r->rn_chunk1.ch_text->tx_fname,
r->rn_size);
#endif
r->rn_next = runs;
runs = r;
}
static void
set_chunk(struct chunk *cnk, struct text *txt,
size_t start, size_t size
) {
/* Fill the chunk *cnk with info about the piece of text
in txt starting at start extending over size tokens.
*/
cnk->ch_text = txt;
set_pos(&cnk->ch_first, 0, txt, start);
set_pos(&cnk->ch_last, 1, txt, start + size - 1);
}
static void
set_pos(struct position *pos, int type, struct text *txt, size_t start) {
/* Fill a single struct position */
pos->ps_next = txt->tx_pos;
txt->tx_pos = pos;
pos->ps_type = type;
pos->ps_tk_cnt = start;
pos->ps_nl_cnt = (size_t) -1; /* uninitialized */
}
/* instantiate sort_run_list(struct run **listhook) */
#define SORT_STRUCT run
#define SORT_NAME sort_run_list
#define SORT_BEFORE(r0,r1) ((r0)->rn_size > (r1)->rn_size)
#define SORT_NEXT rn_next
#include "sortlist.bdy"
static void
reverse_runs(struct run **r_p) {
struct run *r = *r_p;
struct run *res = 0;
while (r) {
struct run *top = r;
r = top->rn_next;
top->rn_next = res;
res = top;
}
*r_p = res;
}
struct run *
unsorted_runs(void) {
reverse_runs(&runs);
return runs;
}
struct run *
sorted_runs(void) {
reverse_runs(&runs);
sort_run_list(&runs);
return runs;
}
void
discard_runs(void) {
while (runs) {
struct run *r = runs;
runs = r->rn_next;
Free(r);
}
}
#ifdef DB_RUN
void
db_run_info(const char *msg, const struct run *run, int lines_too) {
const struct chunk *cnk0 = &run->rn_chunk0;
const struct chunk *cnk1 = &run->rn_chunk1;
if (msg) {
fprintf(Debug_File, "%s: ", msg);
}
fprintf(Debug_File, "\"%s\" / \"%s\":\n",
cnk0->ch_text->tx_fname, cnk1->ch_text->tx_fname
);
fprintf(Debug_File, "from %s %s/%s to %s/%s:", Token_Name,
size_t2string(cnk0->ch_first.ps_tk_cnt),
size_t2string(cnk1->ch_first.ps_tk_cnt),
size_t2string(cnk0->ch_last.ps_tk_cnt),
size_t2string(cnk1->ch_last.ps_tk_cnt)
);
if (lines_too) {
fprintf(Debug_File, " from lines %s/%s to %s/%s:",
size_t2string(cnk0->ch_first.ps_nl_cnt),
size_t2string(cnk1->ch_first.ps_nl_cnt),
size_t2string(cnk0->ch_last.ps_nl_cnt),
size_t2string(cnk1->ch_last.ps_nl_cnt)
);
}
fprintf(Debug_File, " %s %s%s\n",
size_t2string(run->rn_size),
Token_Name, (run->rn_size == 1 ? "" : "s")
);
}
#endif /* DB_RUN */

38
core/sim/sim_3_01/runs.h Normal file
View File

@@ -0,0 +1,38 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: runs.h,v 1.13 2016-07-31 18:55:44 dick Exp $
*/
/* Although all other segments of data in this program are described by
giving the position of the first in the segment and that of the
first not in the segment (so the size is the difference of the two),
a `chunk' is given by first and last. This is done because later on we
are interested in the actual position of the last token of it, and
the position of the first token not in the segment gives no
indication about that.
*/
struct chunk {
/* a chunk of text */
const struct text *ch_text; /* pointer to the file */
struct position ch_first; /* first in chunk */
struct position ch_last; /* last in chunk */
};
struct run { /* a 'run' of coincident tokens */
struct run *rn_next;
struct chunk rn_chunk0; /* chunk in left file */
struct chunk rn_chunk1; /* chunk in right file */
size_t rn_size;
};
extern void add_to_runs(
struct text *txt0, size_t i0, struct text *txt1, size_t i1,
size_t size);
extern struct run *sorted_runs(void);
extern struct run *unsorted_runs(void);
extern void discard_runs(void);
#ifdef DB_RUN
extern void db_run_info(const char *msg, const struct run *run, int lines_too);
#endif /* DB_RUN */

8
core/sim/sim_3_01/settings.par Normal file
View File

@@ -0,0 +1,8 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: settings.par,v 1.6 2016-06-08 09:30:15 dick Exp $
*/
#define DEFAULT_MIN_RUN_SIZE (24)
#define DEFAULT_PAGE_WIDTH (80)

678
core/sim/sim_3_01/sim.1 Executable file
View File

@@ -0,0 +1,678 @@
.\" This file is part of the software similarity tester SIM.
.\" Written by Dick Grune, Vrije Universiteit, Amsterdam.
.\" $Id: sim.1,v 2.40 2017-03-19 09:30:38 dick Exp $
.\"
.TH SIM 1 2016/08/01
.SH NAME
sim \- find similarities in C, Java, Pascal, Modula-2, Lisp, Miranda, or text files
.SH SYNOPSIS
.B sim_c
[
.B \-[adefFiMnOpPRsSTuv]
.B \-r
.I N
.B \-t
.I N
.B \-w
.I N
.B \-o
.I F
]
file ... [ [
.B /
.B |
] file ... ]
.br
.B sim_c++
\&...
.br
.B sim_java
\&...
.br
.B sim_pasc
\&...
.br
.B sim_m2
\&...
.br
.B sim_lisp
\&...
.br
.B sim_mira
\&...
.br
.B sim_text
\&...
.br
.SH DESCRIPTION
.I Sim_c
reads the C files
.I file ...
and looks for segments of text that are similar; two segments of program text
are similar if they only differ in layout, comment, identifiers, and
the contents of numbers, strings and characters.
If any runs of sufficient length
are found, they are reported on standard output; the number of significant
tokens in the run is given between square brackets.
.PP
.I Sim_c++
does the same for C++,
.I sim_java
for Java,
.I sim_pasc
for Pascal,
.I sim_m2
for Modula-2,
.I sim_mira
for Miranda,
and
.I sim_lisp
for Lisp.
.I Sim_text
works on arbitrary text and it is occasionally useful on shell scripts.
.PP
The program can be used for finding copied pieces of code in
purportedly unrelated programs (with
.B \-s
or
.BR \-S ),
or for finding accidentally duplicated code in larger projects (with
.B \-f
or
.BR \-F ).
.PP
If a separator
.B /
or
.B |
is present in the list of input files, the files are divided into a group of
"new" files (before the
.BR /
or
.BR | )
and a group of "old" files; if there is no
.BR /
or
.BR | ,
all files are "new".
Old files are never compared to other files.
See also the description of the
.B \-s
and
.B \-S
options below.
.PP
Since the similarity tester needs file names to pinpoint the similarities, it
cannot read from standard input.
.PP
The similarity tester takes ASCII or UTF-8 text as input, and produces a
sorted list of runs in text form (default or with the
.B -d
or
.B -n
options) or in percentage form (with the
.B -p
option).
Input in other formats, e.g.
.I .pdf
or
.I .doc
needs to be converted to ASCII or UTF-8 by preprocessing.
Aggregated similarity results can be obtained by doing postprocessing on the
output.
.PP
There are the following options:
.TP
.B \-d
The output is in a diff(1)-like format instead of the default
2-column format.
Recommended for text in languages with non-Latin alphabets.
.TP
.B \-e
Each file is compared to each file in isolation. This will find all
similarities between all texts involved, regardless of repetitive text,
but may be slow for large numbers of files.
See also `Calculating Percentages' below.
.TP
.B \-f
Runs are restricted to segments with balancing parentheses, to isolate
potential routine bodies (not in
.IR sim_text ).
.TP
.B \-F
The names of routines in calls are required to match exactly
(not in
.IR sim_text ).
.TP
.B \-i
The names of the files to be compared are read from standard input, including
a possible separator
.BR /
or
.BR | ;
the file names must be one to a line.
This option allows a very large number of file names to be specified;
it differs from the \fC@\fP facility provided by some compilers in that it
handles file names only, and does not recognize option arguments.
.TP
.B \-M
Memory usage information is displayed on standard error output.
.TP
.B \-n
Similarities found are summarized by file name, position and size, rather than
displayed in full.
.TP
.B "\-o F"
The output is written to the file named
.IR F .
.TP
.B \-O
The option settings used are shown at the beginning of the output.
.TP
.B \-p
The output is given in similarity percentages; see `Calculating Percentages'
below; implies \fB\-s\fP.
.TP
.B \-P
When reporting percentages, only the main contributor for each file is shown.
.TP
.B "\-r N"
The minimum run length is set to
.I N
units; the default is 24 tokens, except in
.IR sim_text ,
where it is 8 words.
.TP
.B \-R
Directories in the input list are entered recursively, and all files they
contain are involved in the comparison.
.TP
.B \-s
The contents of a file are not compared to itself (\-s for "not self").
.TP
.B \-S
The contents of the new files are compared to the old files only \- not
between themselves.
.TP
.B "\-t N"
In combination with the
.B \-p
option, sets the threshold (in percents) below which similarities will not be
reported; the default is 1, except in
.IR sim_text ,
where it is 20.
.TP
.B \-T
Suppresses the printing of information about the input files.
.TP
.B \-u
The output is not buffered and not sorted (only when reporting percentages).
.TP
.B \-v
Prints the version number and compilation date on standard output, then stops.
.TP
.B "\-w N"
The page width used is set to
.I N
columns; the default is 80.
.TP
.B "\-\-"
(A secret option, which prints the input as the similarity checker sees it,
and then stops.)
.PP
The
.B \-p
option results in lines of the form
.nf
.ft C
F consists for x % of G material
.ft P
.fi
meaning that \fCx\fP % of \fCF\fP's text can also be found in \fCG\fP.
Note that this relation is not symmetric; it is quite possible for one
file to consist for 100 % of text from another file, while the other file
consists for only 1 % of text of the first file, if their lengths differ
enough.
The
.B \-P
(capital P) option shows the main contributor for each file only.
This simplifies the identification of a set of files \fCA[1] ... A[n]\fP,
where the concatenation of these files is also present.
A threshold can be set using the
.B \-t
option.
Note that the granularity of the recognized text is still governed by the
.B \-r
option or its default.
.PP
The
.B \-r
option controls the number of "units" that constitute a run.
For the programs that compare programming language code, a unit is a lexical
token in the pertinent language; comment and standard preamble material (file
inclusion, etc.) is ignored and all strings are considered equal.
For
.I sim_text
a unit is a "word" which is defined as any sequence of one or more letters,
digits, or characters over 127 (177 octal), to accommodate full UNICODE (UTF-8).
.PP
The programs can handle UNICODE (UTF-16) file names under Windows.
This is relevant only under the
.B \-R
option, since there is no way to supply UNICODE file names from the command
line.
.PP
.I Sim_text
accepts s p a c e d t e x t as normal text.
.PP
Once
.I sim
has read, stored and preprocessed the input, it will no longer run out of
memory.
If memory is short it will change automatically to unbuffered, unsorted
output (while isuuing a warning message).
.SH WHAT IS COMPARED TO WHAT
The default operation cycle of
.I sim
starts at the beginning of the first input file or at a point
.I X
in a file
.I F
among the new files, i.e. those before the new/old separator, if present.
.I Sim
then finds the longest segment
.I S
such that
1)
.I S
is equal to the segment starting at
.IR X ;
2)
.I S
is situated somewhere between position
.I X
in
.I F
and the end of all files;
3)
.I S
does not overlap with the segment starting at
.IR X .
If the segment is at least of minimum run size, it is recorded, and the cycle
starts again just after the segment at
.IR X ;
otherwise it starts again at
.IR X+1 .
.PP
So if the tokens at
.I X
read \fCabcabcadefabdabcz\fP, the cycle finds
.I S
to be the \fCabc\fP just before the end; \fCabda\fP at position 4 would be
longer but overlaps.
The cycle then starts at position 4, and will find another match with the
\fCabc\fP near the end.
Finally the third \fCab\fP be matched with the fourth \fCab\fP just before the
\fCcz\fP.
This way best matches for the text in a file are found in material
to the right of it, until the end of all files.
The results are asymmetric: given files
.IR F1 ,
.IR F2 ,
.IR F3 ,
.IR F4 ,
no matches for
.I F3
are reported from
.I F1
or
.IR F2 ,
for example.
As explained below under "Limitations", this avoids duplicate reports of
similarity and helps to keep
.I sim
fast.
.PP
The area that is searched by
.I sim's
cycle is called the
.IR range .
The default range (running from the file under observation to the end of all
files) is excellent for finding similarities in program files,
and, when doing percentages, for getting an impression of which files are
related to which files, but sometimes more control is needed.
.PP
The
.B \-a
option includes
.I all
text in the range by not stopping the search at the end of the files but
rather looping back to the beginning of the files and continuing to the point
where the search started.
Now matches are also found in files before the present one and the results are
symmetric: given files
.IR F1 ,
.IR F2 ,
.IR F3 ,
.IR F4 ,
matches for
.I F3
will also be reported from
.I F1
or
.IR F2 ,
if present.
But matches may be reported twice, once for file
.I Fa
versus file
.IR Fb ,
and once for file
.I Fb
versus file
.IR Fa .
The
.B \-a
option allows a more accurate determination of similarity percentages.
.PP
The
.B \-a
option is the only way to obtain symmetrical results, with information
about both \fIF1\fP vs. \fIF2\fP and \fIF2\fP vs. \fIF1\fP.
.PP
The
.B \-S
option removes the new files from the range, so files are only compared to the
old files.
.PP
The
.B \-s
option removes the file itself from the range, so a file will not be compared
to itself. This is the default when reporting percentages.
.PP
In normal operation the whole range is searched as one unit. The
.B \-e
option divides up the range into the separate files, and causes
.I sim
to compare a file to each of the other files separately.
This produces the most detailed information when reporting text similarities,
and the best possible results when reporting similarity percentages, but can
be quite slow.
.SS A Tabular Representation
Input files are divided into two groups, new and old.
In the absence of control options
.I sim
compares the files thus (for 4 new files and 6 old ones):
.ne 16
.nf
.ft C
n e w / o l d <- second file
1 2 3 4 / 5 6 7 8 9 10
|------------/------------
n 1 | c c c c / c c c c c c
e 2 | c c c / c c c c c c
w 3 | c c / c c c c c c
4 | c / c c c c c c
first / / / / / / / / / / / / /
file -> 5 | /
o 6 | /
l 7 | /
d 8 | /
9 | /
10 | /
.ft P
.fi
where a \fCc\fP indicates that the first file is compared to the second file,
and the \fC/\fP represents the demarcation between new and old files.
The comparison range of the first files is clearly visible.
.PP
Using the
.B \-a
option extends this to
.ne 16
.nf
.ft C
n e w / o l d <- second file
1 2 3 4 / 5 6 7 8 9 10
|------------/------------
n 1 | c c c c / c c c c c c
e 2 | c c c c / c c c c c c
w 3 | c c c c / c c c c c c
4 | c c c c / c c c c c c
first / / / / / / / / / / / / /
file -> 5 | /
o 6 | /
l 7 | /
d 8 | /
9 | /
10 | /
.ft P
.fi
.PP
Using the
.B \-S
option instead reduces this to
.ne 16
.nf
.ft C
n e w / o l d <- second file
1 2 3 4 / 5 6 7 8 9 10
|------------/------------
n 1 | / c c c c c c
e 2 | / c c c c c c
w 3 | / c c c c c c
4 | / c c c c c c
first / / / / / / / / / / / / /
file -> 5 | /
o 6 | /
l 7 | /
d 8 | /
9 | /
10 | /
.ft P
.fi
.PP
Finally, using the
.B \-s
option changes the default ranges to
.ne 16
.nf
.ft C
n e w / o l d <- second file
1 2 3 4 / 5 6 7 8 9 10
|------------/------------
n 1 | c c c / c c c c c c
e 2 | c c / c c c c c c
w 3 | c / c c c c c c
4 | / c c c c c c
first / / / / / / / / / / / / /
file -> 5 | /
o 6 | /
l 7 | /
d 8 | /
9 | /
10 | /
.ft P
.fi
and the
.BR \-a -extended
ranges to
.ne 16
.nf
.ft C
n e w / o l d <- second file
1 2 3 4 / 5 6 7 8 9 10
|------------/------------
n 1 | c c c / c c c c c c
e 2 | c c c / c c c c c c
w 3 | c c c / c c c c c c
4 | c c c / c c c c c c
first / / / / / / / / / / / / /
file -> 5 | /
o 6 | /
l 7 | /
d 8 | /
9 | /
10 | /
.ft P
.fi
.SH LIMITATIONS
Repetitive input is the bane of similarity checking.
If we have a file containing 4 copies of identical text,
.nf
.ft C
A1 A2 A3 A4
.ft P
.fi
where the numbers serve only to distinguish the identical copies,
there are 7 non-overlapping identities: \fCA1=A2\fP, \fCA1=A3\fP, \fCA1=A4\fP,
\fCA2=A3\fP, \fCA2=A4\fP, \fCA3=A4\fP, and \fCA1A2=A3A4\fP.
Of these, only 3 are meaningful: \fCA1=A2\fP, \fCA2=A3\fP, and \fCA3=A4\fP.
And for a table with 20 lines identical to each other, not unusual in a
program text, there are 715 non-overlapping identities, of which at most 19
are meaningful.
Reporting all 715 of them is clearly unacceptable.
.PP
This is remedied by
.I sim's
search cycle:
for each position in the text, the largest segment is found of which a
non-overlapping copy occurs in the text following it.
That segment and its copy are then reported and scanning resumes at the
position just after the segment.
For the above example this results in the two identities \fCA1A2=A3A4\fP and
\fCA3=A4\fP, which is quite satisfactory, and for \fIN\fP identical segments
roughly \fI2 log N\fP messages are given.
.PP
This also works out well when the four identical segments are in different
files:
,ne 4
.nf
.ft C
File1: A1
File2: A2
File3: A3
File4: A4
.ft P
.fi
Now combined segments like \fCA1A2\fP do not occur, and the algorithm finds
the runs \fCA1=A2\fP, \fCA2=A3\fP, and \fCA3=A4\fP, for a total of \fIN-1\fP
runs, all informative.
.SS Calculating Percentages
The above approach is unsuitable for obtaining the exact percentage of a
file's content that can be found in another file, although indicative results
can be obtained.
Obtaining exact percentages requires comparing each file pair in isolation;
this is what the \fB\-ae\fP options do.
Under the \fB\-ae\fP options a segment \fCFile3:A3\fP, recognized in
\fCFile4\fP, will again be recognized in \fCFile1\fP and \fCFile2\fP.
In the example above it produces the runs
.ne 12
.nf
.ft C
File1:A1=File2:A2
File1:A1=File3:A3
File1:A1=File4:A4
File2:A2=File3:A3
File2:A2=File4:A4
File2:A2=File1:A1
File3:A3=File4:A4
File3:A3=File1:A1
File3:A3=File2:A2
File4:A4=File1:A1
File4:A4=File2:A2
File4:A4=File3:A3
.ft P
.fi
for a total of \fIN(N-1)\fP runs.
.PP
When the
.B \-e
option is used alone.
.I sim
will find the following runs:
.ne 6
.nf
.ft C
File1:A1=File2:A2
File1:A1=File3:A3
File1:A1=File4:A4
File2:A2=File3:A3
File2:A2=File4:A4
File3:A3=File4:A4
.ft P
.fi
for a total of \fI\(12N(N-1)\fP runs, thus missing half the percentage
contributions; in fact, \fCFile4\fP is found to have 0% in common with the
other files.
.PP
If, however, the
.B \-a
option is used alone.
.I sim
finds the following runs:
.ne 4
.nf
.ft C
File1:A1=File2:A2
File2:A2=File3:A3
File3:A3=File4:A4
File4:A4=File1:A1
.ft P
.fi
for a total of \fIN\fP runs. This setting misses many of the percentage
contributions, but finds something for every file.
.SH TIME AND SPACE REQUIREMENTS
Care has been taken to keep the time requirements of all internal processes
(almost) linear in the lengths of the input files, by using various tables.
.PP
The time requirements are quadratic in the number of files.
This means that, for example, one 64 MB file processes much faster than 8000 8
kB files.
.PP
The program requires 6 bytes of memory for each token in the input; 2
bytes per newline (not when doing percentages); and 80 bytes for each
run found.
.SH EXAMPLES
The call
.nf
.ft C
sim_c *.c
.ft P
.fi
highlights duplicate C code in the directory.
(It is useful to remove generated files first.)
A call of
.nf
.ft C
sim_c -f -F *.c
.ft P
.fi
can pinpoint the duplicate code further.
.PP
A call
.nf
.ft C
sim_text -peu -S new/* "|" old/*
.ft P
.fi
compares each file in \fCnew/*\fP to each file in \fCold/*\fP, and if any pair
has more that 20% in common, that fact is reported.
Usually a similarity of 30% or more is significant; lower than 20% is probably
coincidence; and in between is doubtful.
.PP
The \fCu\fP in \fC-peu\fP causes the output to be unbuffered (and unsorted), so
if the program is stopped for running out of time, any results already found
are not lost.
.PP
For large data sets, using \fC-pu\fP rather than \fC-peu\fP may do the job much
more quickly, but less accurately.
.PP
The \fC|\fP can be used as a separator instead of \fC/\fP on systems where the
\fC/\fP as a command-line parameter gets mangled by the command interpreter.
.PP
These calls are good for plagiarism detection.
.SH BUGS
Unbuffered, unsorted output is not available for text output, only for
percentage output.
.SH AUTHOR
Dick Grune, Vrije Universiteit, Amsterdam; dick@dickgrune.com.

253
core/sim/sim_3_01/sim.c Executable file
View File

@@ -0,0 +1,253 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: sim.c,v 2.69 2017-03-19 09:30:38 dick Exp $
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "system.par"
#include "settings.par"
#include "sim.h"
#include "options.h"
#include "newargs.h"
#include "token.h"
#include "tokenarray.h"
#include "language.h"
#include "text.h"
#include "runs.h"
#include "hash.h"
#include "compare.h"
#include "pass1.h"
#include "pass2.h"
#include "pass3.h"
#include "percentages.h"
#include "stream.h"
#include "lang.h"
#include "Malloc.h"
#include "any_int.h"
const char *Version;
/* PARAMETERS */
/* Command-line parameters, with defaults */
int Min_Run_Size = DEFAULT_MIN_RUN_SIZE;
int Page_Width = DEFAULT_PAGE_WIDTH;
int Threshold_Percentage = 1;
FILE *Output_File;
FILE *Debug_File;
/* Language-specific parameters; may be changed in Init_Language() */
const char *Token_Name = "token";
static const char *progname; /* for error reporting */
static const char *output_name; /* for redirecting the output */
static const struct option optlist[] = {
{'r', "set minimum run size to N", Number, &Min_Run_Size},
{' ', "output runs as text (default)", None, 0},
{'d', "output in a diff-like format", None, 0},
{'n', "suppress the text of the runs", None, 0},
{'T', "suppress reporting the input files", None, 0},
{'p', "output similarity in percentages", None, 0},
{'P', "main contributing file to percentages only", None, 0},
{'t', "threshold level of percentages", Number, &Threshold_Percentage},
{'e', "compare each file to each file separately", None, 0},
{' ', "compare a file to files after it only (default)", None, 0},
{'a', "compare to all files", None, 0},
{'S', "compare to old files only", None, 0},
{'s', "do not compare a file to itself", None, 0},
{' ', "sorted output, most significant first (default)", None, 0},
{'u', "unbuffered, unsorted output", None, 0},
{' ', "miscellaneous options:", None, 0},
{'f', "function-like forms only", None, 0},
{'F', "keep function identifiers in tact", None, 0},
{'R', "recurse into subdirectories", None, 0},
{'i', "read arguments (file names) from standard input", None, 0},
{'o', "write output to file F", String, &output_name},
{'w', "set page width to N", Number, &Page_Width},
{'O', "show command line options at start-up", None, 0},
{'M', "show memory usage info at close-down", None, 0},
{'v', "show version number and compilation date", None, 0},
{'-', "lexical scan output only", None, 0},
{0, 0, 0, 0}
};
/* SERVICE ROUTINES */
int
is_new_old_separator(const char *s) {
if (strcmp(s, "/") == 0) return 1;
if (strcmp(s, "|") == 0) return 1;
return 0;
}
const char *
size_t2string(size_t s) {
return any_uint2string(s, 0);
}
void
fatal(const char *msg) {
fprintf(stderr, "%s: %s\n", progname, msg);
exit(1);
}
/* PROGRAM */
#ifdef ARG_TEST
static void
show_args(const char *msg, int argc, const char *argv[]) {
fprintf(stdout, "%s: ", msg);
int i;
for (i = 0; i < argc; i++) {
fprintf(stdout, "arg[%d] = %s; ", i, argv[i]);
}
fprintf(stdout, "\n");
}
#endif /* ARG_TEST */
int
main(int argc, const char *argv[]) {
/* The value of Version derives from the macro VERSION in the
Makefile if present. If not, a build time stamp is created.
*/
char version[40];
#ifdef VERSION
sprintf(version, "Version %s", VERSION);
#else
sprintf(version, "Build %s, %s", __DATE__, __TIME__);
#endif
Version = version;
/* Save program name */
progname = argv[0];
argv++, argc--; /* and skip it */
/* Set the default output and debug streams */
Output_File = stdout;
Debug_File = stdout;
Threshold_Percentage = 1;
/* Options, default string values given above */
/* override from language file */
Init_Language();
/* override from command line */
{ int n_op = do_options(progname, optlist, argc, argv);
argc -= n_op, argv += n_op; /* and skip them */
}
/* Check options compatibility */
allow_at_most_one_option_out_of("dnp"); /* alternative output formats */
allow_at_most_one_option_out_of("aS"); /* alternative ranges */
allow_at_most_one_option_out_of("sS"); /* self is outside old files */
if (is_set_option('t')) {
/* threshold means percentages */
if (!is_set_option('p'))
fatal("option -t requires -p");
}
if (is_set_option('P')) {
if (!is_set_option('p'))
fatal("option -P requires -p");
}
/* ZZ
if (is_set_option('u')) {
if (!is_set_option('p'))
fatal("option -u available with -p only");
}
*/
/* Treat the simple options */
if (is_set_option('v')) {
fprintf(stdout, "%s\n", Version);
return 0;
}
if (is_set_option('p')) {
set_option('s');
}
/* Check the value options */
if (Min_Run_Size <= 0)
fatal("bad run size");
if (Page_Width <= 0)
fatal("bad page width");
if (is_set_option('p')) {
if ((Threshold_Percentage > 100) || (Threshold_Percentage <= 0))
fatal("threshold must be between 1 and 100");
}
if (output_name) {
Output_File = fopen(output_name, "w");
if (Output_File == 0) {
char *msg = (char *)Malloc(strlen(output_name) + 100);
sprintf(msg, "cannot open output file `%s'",
output_name);
fatal(msg);
/*NOTREACHED*/
}
}
/* Treat the input-determining options */
if (is_set_option('i')) {
/* read input file names from standard input */
if (argc != 0)
fatal("-i option conflicts with file arguments");
get_new_std_input_args(&argc, &argv);
}
if (is_set_option('R')) {
get_new_recursive_args(&argc, &argv);
}
/* (argc, argv) now represents new_file* [ / old_file*] */
/* Optionally show command line options */
if (is_set_option('O')) {
print_options(progname, optlist);
}
/* Here the real work starts */
if (is_set_option('-')) {
/* Just the lexical scan */
while (argv[0]) {
const char *arg = argv[0];
if (!is_new_old_separator(arg)) {
Print_Stream(arg);
}
argv++;
}
}
else { /* The works */
Read_Input_Files(argc, argv);
Make_Forward_References();
Compare_Files();
Free_Forward_References();
if (is_set_option('p')) {
Show_Percentages();
} else {
Retrieve_Runs();
Show_Runs();
}
}
Free_Text();
Free_Token_Array();
if (is_set_option('M')) {
ReportMemoryStatus(stderr);
}
return 0;
}

27
core/sim/sim_3_01/sim.h Executable file
View File

@@ -0,0 +1,27 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: sim.h,v 2.22 2016-05-29 15:23:16 dick Exp $
*/
#include <stdio.h>
extern const char *Version;
extern int Min_Run_Size;
extern int Page_Width;
extern FILE *Output_File;
extern FILE *Debug_File;
extern const char *Token_Name; /* possibly modified in *lang.l */
extern int Threshold_Percentage;
/* Service routines */
extern int is_new_old_separator(const char *s);
extern const char *size_t2string(size_t s);
extern void fatal(const char *msg);
/* All output goes through designated files, so we block printf, etc. */
#undef printf
#define printf use_fprintf
#undef putchar
#define putchar use_fprintf

BIN
core/sim/sim_3_01/sim.pdf Executable file
View File

Binary file not shown.

64
core/sim/sim_3_01/sortlist.bdy Normal file
View File

@@ -0,0 +1,64 @@
/*
Module: Sort Linked Lists
Author: dick@dickgrune.com (Dick Grune, Amstelveen)
Version: 2015-01-18
Description:
This is the implementation part of a generic routine that sorts
linked lists.
Instantiation:
See sortlist.spc
*/
#ifndef _SORT_EXTERN_DEFINED
static
#endif
void
SORT_NAME(struct SORT_STRUCT **l_hook) {
/* by split-sort-merge */
struct SORT_STRUCT *lst = *l_hook;
if (lst == 0) return; /* the empty list is sorted */
if (lst->SORT_NEXT == 0) return; /* a 1-element list is sorted */
/* There are at least two elements; split them into two sublists. */
struct SORT_STRUCT *q0 = 0, *q1 = 0; /* starts of the sublists */
struct SORT_STRUCT **q_hook[2]; /* append hooks for the lists */
q_hook[0] = &q0, q_hook[1] = &q1;
int q_cnt = 0; /* pertinemt sublist pointer */
while (lst) {
/* Detach the head element */
struct SORT_STRUCT *l = lst;
lst = lst->SORT_NEXT;
l->SORT_NEXT = 0;
/* and append it to the pertinent sublist. */
*q_hook[q_cnt] = l;
q_hook[q_cnt] = &l->SORT_NEXT;
q_cnt = 1 - q_cnt; /* switch pertinent sublist */
}
/* Sort recursively. */
SORT_NAME(&q0);
SORT_NAME(&q1);
/* Merge. */
*l_hook = 0;
while (q0 || q1) {
/* determine the list with the smallest head element */
struct SORT_STRUCT **h_hook = (
q0 == 0 ? &q1 :
q1 == 0 ? &q0 :
SORT_BEFORE((q0), (q1)) ? &q0 : &q1
);
/* detach head element */
struct SORT_STRUCT *l = *h_hook;
*h_hook = (*h_hook)->SORT_NEXT;
l->SORT_NEXT = 0;
/* append l to l_hook */
*l_hook = l;
l_hook = &l->SORT_NEXT;
}
}

64
core/sim/sim_3_01/sortlist.spc Normal file
View File

@@ -0,0 +1,64 @@
/*
Module: Sort Linked Lists
Author: dick@cs.vu.nl (Dick Grune @ Vrije Universiteit, Amsterdam)
Version: 2015-01-18
Description:
This is the specification part of a generic routine that sorts linked
lists. The elements in the list are structs, each of which carries a
pointer to the next element.
Specification:
The module supplies:
- a routine void SORT_NAME(struct SORT_STRUCT **listhook)
where 'listhook' is a pointer to the location that holds the
pointer to the list to be sorted. Upon return, the list will
be sorted, and the pointer updated.
The routine will be defined static when instantiated inline.
Instantiation, inline:
For each struct list type T, specify:
- a definition of SORT_STRUCT, the struct name of the linked
structs;
- a definition of SORT_NAME, the name of the resulting sort
routine;
- a definition of a routine
int SORT_BEFORE(
struct SORT_STRUCT *v, struct SORT_STRUCT *w
)
or a definition
#define SORT_BEFORE((v,w)
which yields non-zero if v is to be sorted before w;
- a definition of a field selector SORT_NEXT which names the
field that points to the next struct SORT_STRUCT in the list.
- #include "sortlist.bdy"
Instantiation, separate:
For each struct list type T, create a file sort_T.h which contains at
least:
- a definition of SORT_STRUCT, the struct name of the linked
structs;
- a definition of SORT_NAME, the name of the resulting sort
routine;
- #include "sortlist.spc"
This file sort_T.h is to be included in all files that use the routine
SORT_NAME.
For each struct list type T, create a file sort_T.c which contains at
least:
- #include "sort_T.h"
- a definition of a routine or definition SORT_BEFORE as
described above;
- a definition of a field selector SORT_NEXT which names the
field that points to the next struct SORT_STRUCT in the list;
- #include "sortlist.bdy"
This file sort_T.c compiles into the module object for SORT_STRUCT.
Implementation:
Recursive split-sort-merge.
*/
extern void SORT_NAME(struct SORT_STRUCT **);
#define _SORT_EXTERN_DEFINED

91
core/sim/sim_3_01/stream.c Normal file
View File

@@ -0,0 +1,91 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: stream.c,v 2.14 2016-05-13 19:00:54 dick Exp $
*/
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "system.par"
#include "sim.h"
#include "options.h"
#include "token.h"
#include "lang.h"
#include "stream.h"
static FILE *fopen_regular_file(const char *fname);
int
Open_Stream(const char *fname) {
int ok;
lex_nl_cnt = 1;
lex_tk_cnt = 0;
lex_non_ascii_cnt = 0;
/* start the lex machine */
yyin = fopen_regular_file(fname);
ok = (yyin != 0);
if (!ok) {
/* fake a stream, to simplify the rest of the program */
yyin = fopen(NULLFILE, "r");
}
yystart();
return ok;
}
static FILE *
fopen_regular_file(const char *fname) {
struct stat buf;
if (stat(fname, &buf) != 0) return 0;
if ((buf.st_mode & S_IFMT) != S_IFREG) return 0;
return fopen(fname, "r");
}
int
Next_Stream_Token_Obtained(void) {
return yylex();
}
void
Close_Stream(void) {
if (yyin) {
fclose(yyin);
yyin = 0;
}
}
void
Print_Stream(const char *fname) {
fprintf(Output_File, "File %s:", fname);
if (!Open_Stream(fname)) {
fprintf(Output_File, " cannot open\n");
return;
}
fprintf(Output_File, " showing the %s stream\n", Token_Name);
lex_token = End_Of_Line;
do {
if (Token_EQ(lex_token, End_Of_Line)) {
fprintf(Output_File,
"line # = %s, %s # = %s:\n",
size_t2string(lex_nl_cnt),
Token_Name,
size_t2string(lex_tk_cnt)
);
}
else {
extern char *yytext;
fprintf(Output_File, " %s -> ", yytext);
fprint_token(Output_File, lex_token);
fprintf(Output_File, "\n");
}
} while (Next_Stream_Token_Obtained());
fprintf(Output_File, "\n");
Close_Stream();
}

16
core/sim/sim_3_01/stream.h Normal file
View File

@@ -0,0 +1,16 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: stream.h,v 2.7 2012-06-08 16:04:30 dick Exp $
*/
/*
Interface of the stream module.
Implements the direct interaction with the lexical
module. It supplies the routines below.
*/
extern int Open_Stream(const char *);
extern int Next_Stream_Token_Obtained(void);
extern void Close_Stream(void);
extern void Print_Stream(const char *fname);

18
core/sim/sim_3_01/system.par Normal file
View File

@@ -0,0 +1,18 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: system.par,v 1.3 2014-01-26 13:51:27 dick Exp $
*/
/* Operating-system dependent data */
#ifdef MSDOS /* GNU gcc */
#define int32 int /* type of a 32 bits signed int */
#define NULLFILE "nul"
# else /* various *NIXes */
#define int32 int /* type of a 32 bits signed int */
#define NULLFILE "/dev/null"
#endif

256
core/sim/sim_3_01/text.c Normal file
View File

@@ -0,0 +1,256 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: text.c,v 1.23 2016-08-03 19:14:04 dick Exp $
*/
#include <stdio.h>
#include <stdlib.h>
#include "debug.par"
#include "sim.h"
#include "token.h"
#include "stream.h"
#include "lang.h"
#include "Malloc.h"
#include "options.h"
#include "text.h"
struct text *Text; /* to be filled in by Malloc() */
int Number_of_Texts;
int Number_of_New_Texts;
typedef unsigned short nl_tk_diff_t;
struct newline {
nl_tk_diff_t nl_tk_diff; /* token position difference */
};
#define NL_START 1024 /* initial newline buffer size */
static struct newline *nl_buff; /* to be filled by Malloc() */
static size_t nl_size; /* size of nl_buff[] */
static size_t nl_free; /* next free position in nl_buff[] */
static size_t nl_next, nl_limit; /* nl_buff[] pointers during pass 2 */
static void store_newline(void);
static void init_nl_buff(void);
/* TEXT INTERFACE */
static size_t last_tk_cnt; /* token count at newline */
static size_t last_nl_cnt; /* nl counter during pass 2 */
void
Init_Text(int nfiles) {
/* allocate the array of text descriptors */
if (Text) {
Free(Text);
Text = 0;
}
Number_of_Texts = nfiles;
Text = (struct text *)
Malloc((size_t)(Number_of_Texts*sizeof (struct text)));
init_nl_buff();
}
int
Open_Text(enum Pass pass, struct text *txt) {
switch (pass) {
case First_Pass:
last_tk_cnt = 0;
if (nl_buff) {
txt->tx_nl_start = nl_free;
}
break;
case Second_Pass:
last_tk_cnt = 0;
if (nl_buff) {
nl_next = txt->tx_nl_start;
nl_limit = txt->tx_nl_limit;
last_nl_cnt = 1;
lex_nl_cnt = 1;
lex_tk_cnt = 0;
return 1;
}
break;
}
return Open_Stream(txt->tx_fname);
}
int
Next_Text_Token_Obtained(void) {
if (!Next_Stream_Token_Obtained()) return 0;
if (Token_EQ(lex_token, End_Of_Line)) {
store_newline();
last_tk_cnt = lex_tk_cnt;
}
return 1;
}
int
Next_Text_EOL_Obtained(void) {
/* get newline info from the buffer or from the file itself */
if (nl_buff) {
if (nl_next == nl_limit) return 0;
struct newline *nl = &nl_buff[nl_next++];
lex_nl_cnt = ++last_nl_cnt;
lex_tk_cnt = (last_tk_cnt += nl->nl_tk_diff);
lex_token = End_Of_Line;
return 1;
} else {
int ok;
while ( (ok = Next_Stream_Token_Obtained())
&& !Token_EQ(lex_token, End_Of_Line)
) {
/* skip */
}
return ok;
}
}
void
Close_Text(enum Pass pass, struct text *txt) {
switch (pass) {
case First_Pass:
if (nl_buff) {
if (last_tk_cnt != lex_tk_cnt) {
/* there were tokens after the last newline */
store_newline();
}
txt->tx_nl_limit = nl_free;
}
break;
case Second_Pass:
break;
}
Close_Stream();
}
void
Free_Text(void) {
if (nl_buff) {
Free(nl_buff); nl_buff = 0;
}
if (Text) {
Free(Text); Text = 0;
}
}
/* NEWLINE CACHING */
/* To speed up pass2 which is interested in token positions at line ends,
the newline buffer keeps this info from pass1. To reduce the size of
the newline buffer, the info is kept as the differences of the values
at consecutive line ends. This allows unsigned chars to be used rather
than integers.
The recording of token position differences at End_Of_Line is optional,
and is switched off if
- there is not room enough for the newline buffer;
- a difference would not fit in the field in the struct;
- we are reporting percentages.
Switching off is done by freeing the buffer and setting nl_buff to 0.
Anybody using nl_buff should therefore test for nl_buff being zero.
*/
static void abandon_nl_buff(const char *);
static void
init_nl_buff(void) {
/* if we are doing percentages, we don't need the nl_buff mechanism */
if (is_set_option('p')) return;
/* Allocate the newline buffer, if possible */
nl_size = 0 + NL_START;
nl_buff = (struct newline *)TryMalloc(sizeof (struct newline)*nl_size);
nl_free = 0;
}
static void
store_newline(void) {
if (!nl_buff) return;
if (nl_free == nl_size) {
/* allocated array is full; try to increase its size */
size_t new_size = nl_size + nl_size/2;
if (new_size < nl_free) {
abandon_nl_buff("out of address space");
return;
}
struct newline *new_buff = (struct newline *)TryRealloc(
(char *)nl_buff,
sizeof (struct newline) * new_size
);
if (!new_buff) {
abandon_nl_buff("out of memry");
return;
}
nl_buff = new_buff, nl_size = new_size;
}
/* now we are sure there is room enough */
{
struct newline *nl = &nl_buff[nl_free++];
size_t tk_diff = lex_tk_cnt - last_tk_cnt;
nl->nl_tk_diff = (nl_tk_diff_t) tk_diff;
if (nl->nl_tk_diff != tk_diff) {
abandon_nl_buff("tk_diff does not fit in nl_tk_diff");
}
}
}
static void /*ARGSUSED*/
abandon_nl_buff(const char *msg) {
#undef DB_BUFF
#ifdef DB_BUFF
fprintf(Debug_File, "abandon_nl_buff, %s\n", msg);
#endif /* DB_BUFF */
if (nl_buff) {
Free(nl_buff);
nl_buff = 0;
}
}
#ifdef DB_NL_BUFF
void
db_print_nl_buff(size_t start, size_t limit) {
size_t i;
fprintf(Debug_File, "\n**** DB_NL_BUFF ****\n");
if (!nl_buff) {
fprintf(Debug_File, ">>>> NO NL_BUFF\n\n");
return;
}
if (start > nl_free) {
fprintf(Debug_File, ">>>> start (%s) > nl_free (%s)\n\n",
size_t2string(start), size_t2string(nl_free)
);
return;
}
if (limit > nl_free) {
fprintf(Debug_File, ">>>> limit (%s) > nl_free (%s)\n\n",
size_t2string(limit), size_t2string(nl_free)
);
return;
}
fprintf(Debug_File, "nl_buff: %s entries:\n", size_t2string(nl_free));
for (i = start; i < limit; i++) {
struct newline *nl = &nl_buff[i];
fprintf(Debug_File, "nl_tk_diff = %d\n", nl->nl_tk_diff);
}
fprintf(Debug_File, "\n");
}
#endif /* DB_NL_BUFF */

57
core/sim/sim_3_01/text.h Normal file
View File

@@ -0,0 +1,57 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: text.h,v 1.9 2016-07-28 07:00:48 dick Exp $
*/
/* Implements the access to the lexical scanner.
Additionally, the module tries to save newline information,
anticipating a second scan which is interested in this
information only.
*/
/* The input files are called "texts" */
struct text {
const char *tx_fname; /* the file name */
size_t tx_start; /* index of first token in Token_Array[]
belonging to the text */
size_t tx_limit; /* index of first position in Token_Array[]
not belonging to the text */
size_t tx_nl_start; /* possibly newline pointer for pass2 */
size_t tx_nl_limit;
int tx_EOL_terminated; /* Boolean */
struct position *tx_pos;/* list of positions in this file that are
part of a chunk; sorted and updated by
Pass 2
*/
};
struct position {
/* position of first and last token of a chunk */
struct position *ps_next;
int ps_type; /* first = 0, last = 1 */
size_t ps_tk_cnt; /* in tokens; set by add_run()
in Read_Input_Files() */
size_t ps_nl_cnt; /* same, in line numbers;set by Retrieve_Runs(),
used by Show_Runs(), to report line numbers
*/
};
extern struct text *Text; /* Text[], one for each input file */
extern int Number_of_Texts; /* number of text files;
this includes the new/old separator
if present; actually a design flaw ZZ
*/
extern int Number_of_New_Texts; /* number of new text files */
extern void Init_Text(int nfiles);
enum Pass {First_Pass, Second_Pass};
extern int Open_Text(enum Pass pass, struct text *txt);
extern int Next_Text_Token_Obtained(void);
extern int Next_Text_EOL_Obtained(void);
extern void Close_Text(enum Pass pass, struct text *txt);
extern void Free_Text(void);
#ifdef DB_NL_BUFF
extern void db_print_nl_buff(size_t start, size_t limit);
#endif

95
core/sim/sim_3_01/textlang.l Normal file
View File

@@ -0,0 +1,95 @@
%{
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: textlang.l,v 1.21 2016-05-13 19:00:55 dick Exp $
*/
/*
Text front end for the similarity tester.
*/
#include "sim.h"
#include "token.h"
#include "idf.h"
#include "lex.h"
#include "lang.h"
#include "language.h"
/* General language front end data */
Token lex_token;
size_t lex_nl_cnt;
size_t lex_tk_cnt;
size_t lex_non_ascii_cnt;
/* Language-dependent code */
const char *Subject = "text";
void
Init_Language(void) {
if (is_set_option('f') || is_set_option('F'))
fatal("options -f or -F not applicable in sim_text");
Token_Name = "word";
Threshold_Percentage = 20;
}
/*ARGSUSED*/
int
May_Be_Start_Of_Run(Token tk) {
/* any token is acceptable */
return 1;
}
/*ARGSUSED*/
size_t
Best_Run_Size(const Token *str, size_t size) {
/* any run size is acceptable */
return size;
}
static Token
word2token(char *word) {
/* ignore case */
lower_case(word);
return idf_hashed(word);
}
%}
%option noyywrap
WordElem ([a-zA-Z0-9\200-\377])
TightWord ({WordElem}+)
NonWordElem ([^a-zA-Z0-9\200-\377])
LooseElem ({WordElem}(" "))
SpacedWord ({LooseElem}+{WordElem})
%%
{TightWord} {
return_tk(word2token(yytext));
}
{SpacedWord}/{NonWordElem} {
/* the / operator works at the top level only */
return_tk(word2token(yytext));
}
\n { /* count newlines */
return_eol();
}
. { /* ignore the rest */
}
%%
/* More language-dependent code */
void
yystart(void) {
BEGIN INITIAL;
}

102
core/sim/sim_3_01/token.c Normal file
View File

@@ -0,0 +1,102 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: token.c,v 2.13 2016-05-01 09:52:55 dick Exp $
*/
/*
Token interface, implementation part.
*/
#include <stdio.h>
#include "token.h"
static int
Token_in_range(const Token tk, int low, int high) {
int tki = Token2int(tk);
if (tki < low) return 0;
if (tki > high) return 0;
return 1;
}
static int
check_and_print(
FILE *ofile, const char *name, int ch, int low, int high, int offset
) {
int ch1 = ch + offset;
if (low <= ch1 && ch1 <= high) {
fprintf(ofile, "%s(%c)", name, (char)ch1);
return 1;
}
return 0;
}
#define is_simple_token(tk) (Token_in_range(tk, 0x0001, 0x00FF))
#define is_CTRL_token(tk) (Token_in_range(tk, 0x0101, 0x011E))
#define is_NORM_token(tk) (Token_in_range(tk, 0x0121, 0x017E))
#define is_MTCT_token(tk) (Token_in_range(tk, 0x0181, 0x019E))
#define is_META_token(tk) (Token_in_range(tk, 0x01A1, 0x01FE))
#define is_hashed_token(tk) (Token_in_range(tk, 0x0200, 0xFFFE))
void
fprint_token(FILE *ofile, const Token tk) {
/* Prints a regular token in two characters:
normal char meta (bit 9 set)
^A cntl $A meta-cntl
A printable #A meta
and hashed tokens in hexadecimal.
*/
int tki = Token2int(tk);
int ch = tki & 0x7F;
int bit8 = tki & 0x80;
if (Token_EQ(tk, No_Token)) {fprintf(ofile, "--"); return;}
if (Token_EQ(tk, IDF)) {fprintf(ofile, "IDF"); return;}
if (Token_EQ(tk, End_Of_Line)) {fprintf(ofile, "EOL"); return;}
if (is_simple_token(tk)) {
if ('!' <= ch && ch <= '~') {
fprintf(ofile, "%s%c", (bit8 ? "8" : ""), ch);
return;
}
if (0 < ch && ch <= ' ') {
fprintf(ofile, "%s%c", (bit8 ? "$" : "^"), ch + '@');
return;
}
if (ch == 0x7F) {
fprintf(ofile, "%s%c", (bit8 ? "$" : "^"), '?');
return;
}
}
if (is_CTRL_token(tk)) {
if (check_and_print(ofile, "CTRL", ch, 'A', '~', '@')) return;
}
if (is_NORM_token(tk)) {
if (check_and_print(ofile, "NORM", ch, '!', '~', '\0')) return;
}
if (is_MTCT_token(tk)) {
if (check_and_print(ofile, "MTCT", ch, 'A', '~', '@')) return;
}
if (is_META_token(tk)) {
if (check_and_print(ofile, "META", ch, '!', '~', '\0')) return;
}
if (is_hashed_token(tk)) {
fprintf(ofile, "0x%04x", tki);
return;
}
/* gap token! */
fprintf(ofile, "!0x%04x!", tki);
}
int
Token_EQ(const Token t1, const Token t2) {
/* to make sure Token_EQ is indeed called with two Token parameters */
return Token2int(t1) == Token2int(t2);
}

74
core/sim/sim_3_01/token.h Normal file
View File

@@ -0,0 +1,74 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: token.h,v 2.13 2017-02-04 16:58:54 dick Exp $
*/
/*
Token interface.
Since the definition of a token has been a continual source of
problems, it is now defined as an ADT 'Token'.
There are four classes of tokens:
1. simple tokens; they derive directly from input characters;
2. summary tokens; they summarise keywords, etc.;
3. special tokens: No_Token, IDF, and End_Of_Line;
4. hashed tokens, segments condensed by idf_hashed().
The first three classes are called 'regular tokens'.
There are also a few 'gap' tokens, tokens not produced by the above
mechanisms, for example 0x100.
In addition to the type Token and the special tokens, the module
defines
1. the constants
N_REGULAR_TOKENS number of regular tokens
N_TOKENS total number of tokens, including No_Token
2. macros for defining summary tokens (with ranges of their parameters):
CTRL(ch) ch in 'A'-'~'
NORM(ch) ch in '!'-'~'
MTCT(ch) ch in 'A'-'~'
META(ch) ch in '!'-'~'
These restrictions are not checked.
3. the conversion routines
Token2int(c)
int2Token(i)
*/
#include <stdio.h>
#ifndef _TOKEN_H
#define _TOKEN_H
#ifdef lint_test
/* For security we want to distinguish tokens from integers. Lint is not
good at this, so for checking we use a pointer to a weird data type
*/
struct for_lint_only {int i;};
typedef struct for_lint_only *Token;
#else /* if normal */
typedef unsigned short Token;
#endif /* lint_test/normal */
#define N_TOKENS (1<<16)
#define N_REGULAR_TOKENS (1<<9)
/* Macros for the composition of tokens */ /* range (gaps unused)*/
#define No_Token int2Token(0) /* 0x0000 */
/* UTF-8 characters */ /* 0x0001-0x00FF */
#define CTRL(ch) int2Token(0x100|((ch)&0x01F)) /* 0x0101-0x011E */
#define NORM(ch) int2Token(0x100|((ch)&0x07F)) /* 0x0121-0x017E */
#define IDF int2Token(0x180) /* 0x0180 */
#define MTCT(ch) int2Token(0x180|((ch)&0x01F)) /* 0x0181-0x019E */
#define META(ch) int2Token(0x180|((ch)&0x07F)) /* 0x01A1-0x01FE */
/* tokens from idf_hashed() */ /* 0x0200-0xFFFE */
#define End_Of_Line int2Token(0xFFFF) /* 0xFFFF */
/* Conversion routines */
#define Token2int(c) ((int)(c))
#define int2Token(i) ((Token)(i))
/* Auxiliaries */
#define is_regular_token(tk) (Token2int(tk) < N_REGULAR_TOKENS)
extern int Token_EQ(const Token t1, const Token t2);
extern void fprint_token(FILE *ofile, const Token tk);
#endif /* _TOKEN_H */

63
core/sim/sim_3_01/tokenarray.c Normal file
View File

@@ -0,0 +1,63 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: tokenarray.c,v 1.22 2016-05-29 15:26:50 dick Exp $
*/
#include <stdio.h>
#include <stdlib.h>
#include "sim.h"
#include "Malloc.h"
#include "token.h"
#include "lang.h"
#include "tokenarray.h"
#define Initial_Token_Array_Size 16384
Token *Token_Array; /* to be filled by Malloc() */
static size_t tk_size; /* size of Token_Array[] */
static size_t tk_free; /* next free position in Token_Array[]*/
void
Init_Token_Array(void) {
if (Token_Array) Free(Token_Array);
tk_size = Initial_Token_Array_Size;
Token_Array = (Token *)Malloc(sizeof (Token) * tk_size);
tk_free = 1; /* don't use position 0 */
}
void
Store_Token(Token tk) {
if (tk_free == tk_size) {
/* allocated array is full; try to increase its size */
size_t new_size = tk_size + tk_size/2;
if (new_size < tk_free)
fatal("out of address space");
Token *new_array =
(Token *)TryRealloc(
(char *)Token_Array, sizeof (Token) * new_size
);
if (!new_array) {
/* we failed */
fatal("out of memory: too much text");
}
Token_Array = new_array, tk_size = new_size;
}
/* now we are sure there is room enough */
Token_Array[tk_free++] = tk;
}
void
Free_Token_Array(void) {
if (Token_Array) {
Free(Token_Array); Token_Array = 0;
}
}
size_t
Token_Array_Length(void) {
return tk_free;
}

13
core/sim/sim_3_01/tokenarray.h Normal file
View File

@@ -0,0 +1,13 @@
/* This file is part of the software similarity tester SIM.
Written by Dick Grune, Vrije Universiteit, Amsterdam.
$Id: tokenarray.h,v 1.7 2016-04-30 18:53:33 dick Exp $
*/
/* Interface for the token storage */
extern void Init_Token_Array(void);
extern void Store_Token(Token tk);
extern void Free_Token_Array(void);
extern size_t Token_Array_Length(void); /* also first free token position */
extern Token *Token_Array;

13
core/sim/sim_3_01/utf8test.c Normal file
View File

@@ -0,0 +1,13 @@
#include <stdio.h>
main() {
int 가;
printf("Insurance Review & Assessment Service, 2010) ©≠𒌩우리", __TIMESTAMP__);
printf("나라에서 노인 부양은 핵가족화와 여성의 사회 활동", sizeof (unsigned short int));
printf("참여가 증가하고 가정에서 노인을 돌볼 수 없어 장기", sizeof (unsigned int));
printf("요양 기관에서 간호와 관리를 하게 되는 경향이 증가", sizeof (unsigned long int));
printf("unsigned long long int %u\n", sizeof (unsigned long long int));
printf("%llu\n", (unsigned long long int)400LLU);
return 0;
}

80
docker/hustoj-cpp/Dockerfile Normal file
View File

@@ -0,0 +1,80 @@
#FROM ubuntu:trusty-20171117 #638MB
#COPY ./sources.list /etc/apt/sources.list
FROM debian:jessie-slim
#COPY ./sources.debian.list /etc/apt/sources.list
COPY ./docker-entrypoint.sh /usr/local/bin/
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
git \
make flex g++ libmysqlclient-dev libmysql++-dev \
php5-fpm php5-mysql php5-gd \
# php5-memcache memcached \
nginx \
# fp-compiler \
# openjdk-7-jdk \
# clang
# mysql
&& echo 'mysql-server-5.5 mysql-server/root_password password ""' | debconf-set-selections \
&& echo 'mysql-server-5.5 mysql-server/root_password_again password ""' |debconf-set-selections \
#&& apt-get update
&& apt-get install -y mysql-server \
# code
&& /usr/sbin/useradd -m -u 1536 judge \
&& cd / && git clone https://github.com/zhblue/hustoj.git \
&& mv /hustoj/trunk /home/judge/src \
# clear
&& rm -R /hustoj \
&& apt-get autoremove -y --purge git \
&& rm -rf /var/lib/apt/lists/* \
#
&& USER=`cat /etc/mysql/debian.cnf |grep user|head -1|awk '{print $3}'` \
&& PASSWORD=`cat /etc/mysql/debian.cnf |grep password|head -1|awk '{print $3}'` \
&& CPU=`grep "cpu cores" /proc/cpuinfo |head -1|awk '{print $4}'` \
&& cd /home/judge/ \
&& mkdir etc data log \
&& cp src/install/java0.policy /home/judge/etc \
&& cp src/install/judge.conf /home/judge/etc \
&& mkdir run0 run1 run2 run3 \
&& chown judge run0 run1 run2 run3 \
&& sed -i "s/OJ_USER_NAME=root/OJ_USER_NAME=$USER/g" etc/judge.conf \
&& sed -i "s/OJ_PASSWORD=root/OJ_PASSWORD=$PASSWORD/g" etc/judge.conf \
&& sed -i "s/OJ_RUNNING=1/OJ_RUNNING=$CPU/g" etc/judge.conf \
&& sed -i "s/DB_USER=\"root\"/DB_USER=\"$USER\"/g" src/web/include/db_info.inc.php \
&& sed -i "s/DB_PASS=\"root\"/DB_PASS=\"$PASSWORD\"/g" src/web/include/db_info.inc.php \
&& chown www-data src/web/upload data \
&& sed -i "s/post_max_size = 8M/post_max_size = 80M/g" /etc/php5/fpm/php.ini \
&& sed -i "s/upload_max_filesize = 2M/upload_max_filesize = 80M/g" /etc/php5/fpm/php.ini \
# judged
&& cd /home/judge/src/core/judged \
&& make \
&& chmod +x judged \
&& cp judged /usr/bin \
&& cd ../judge_client \
&& make \
&& chmod +x judge_client \
&& cp judge_client /usr/bin \
&& cd /home/judge/ \
# db
&& chown -R mysql:mysql /var/lib/mysql \
&& service mysql restart \
&& service mysql status \
#&& USER=`cat /etc/mysql/debian.cnf |grep user|head -1|awk '{print $3}'` \
#&& PASSWORD=`cat /etc/mysql/debian.cnf |grep password|head -1|awk '{print $3}'` \
&& mysql -h localhost -u$USER -p$PASSWORD< /home/judge/src/install/db.sql \
&& echo "insert into jol.privilege values('admin','administrator','N');"|mysql -h localhost -u$USER -p$PASSWORD \
#&& service mysql stop \
&& chmod +x /usr/local/bin/docker-entrypoint.sh \
&& ln -s /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh
COPY nginx/default.conf /etc/nginx/sites-available/default
WORKDIR /home/judge/
EXPOSE 80
VOLUME ["/data"]
ENTRYPOINT ["/docker-entrypoint.sh"]
#CMD judged /home/judge debug && tail -f /dev/null

5
docker/hustoj-cpp/build-clear.sh Executable file
View File

@@ -0,0 +1,5 @@
docker rm $(docker ps -a -q)
docker ps -a | grep "Exited" | awk '{print $1 }'|xargs docker stop
docker ps -a | grep "Exited" | awk '{print $1 }'|xargs docker rm
docker images|grep none|awk '{print $3 }'|xargs docker rmi
echo done.

10
docker/hustoj-cpp/build-dokcer.sh Executable file
View File

@@ -0,0 +1,10 @@
#docker rm -f -v /hustoj-dev
#docker build -f Dockerfile -t hustoj-cpp ./
docker rm -f -v /hustoj-cpp
#docker run -d -it --privileged --name hustoj-cpp -p 8080:80 hustoj-cpp
docker rm -f /hustoj-cpp
docker run -d -it --privileged --name hustoj-cpp -p 8080:80 -v /home/hustoj-cpp/:/data/ hustoj-cpp
#docker exec -i -t hustoj-cpp /bin/bash

55
docker/hustoj-cpp/docker-entrypoint.sh Normal file
View File

@@ -0,0 +1,55 @@
#!/bin/bash
DIRECTORY="/data/data/"
if [ ! -d $DIRECTORY ]; then
mv /home/judge/data/ /data/
else
rm -R /home/judge/data/
fi
ln -s $DIRECTORY /home/judge/data
DIRECTORY="/data/judge.conf"
if [ ! -f $DIRECTORY ]; then
mv /home/judge/etc/judge.conf /data/
else
rm /home/judge/etc/judge.conf
fi
ln -s $DIRECTORY /home/judge/etc/judge.conf
DIRECTORY="/data/db_info.inc.php"
if [ ! -f $DIRECTORY ]; then
mv /home/judge/src/web/include/db_info.inc.php /data/
else
rm /home/judge/src/web/include/db_info.inc.php
fi
ln -s $DIRECTORY /home/judge/src/web/include/db_info.inc.php
DIRECTORY="/data/mysql"
if [ ! -d $DIRECTORY ]; then
mv /var/lib/mysql /data
else
rm -R /var/lib/mysql
fi
ln -s $DIRECTORY /var/lib/mysql
chmod 775 -R /data/data
chgrp -R www-data /data/data
chmod 770 -R /data/upload
chgrp -R www-data /data/upload
chmod 770 -R /data/judge.conf
chgrp -R www-data /data/judge.conf
chmod 770 -R /data/db_info.inc.php
chgrp -R www-data /data/db_info.inc.php
#chown -R mysql:mysql /var/lib/mysql
chown -R mysql:mysql /data/mysql/
service mysql start
/usr/bin/judged
php5-fpm
service nginx start
/bin/bash
exit 0

19
docker/hustoj-cpp/nginx/default.conf Normal file
View File

@@ -0,0 +1,19 @@
server {
listen 80 default_server;
root /home/judge/src/web;
server_name _;
index index.php;
location / {
try_files $uri $uri/ =404;
}
location ~ \.php$ {
client_max_body_size 100M;
fastcgi_index index.php;
include fastcgi_params;
fastcgi_param SERVER_NAME $http_host;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
#fastcgi_pass 127.0.0.1:9000;
fastcgi_pass unix:/var/run/php5-fpm.sock;
}
}

4
docker/hustoj-cpp/sources.debian.list Normal file
View File

@@ -0,0 +1,4 @@
deb http://mirrors.aliyun.com/debian/ jessie main non-free contrib
deb http://mirrors.aliyun.com/debian/ jessie-proposed-updates main non-free contrib
deb-src http://mirrors.aliyun.com/debian/ jessie main non-free contrib
deb-src http://mirrors.aliyun.com/debian/ jessie-proposed-updates main non-free contrib

Some files were not shown because too many files have changed in this diff Show More