/*	This file is part of the software similarity tester SIM.
	Written by Dick Grune, Vrije Universiteit, Amsterdam.
	$Id: hash.c,v 2.39 2017-02-04 16:58:54 dick Exp $
*/

/*	Text is compared by comparing every substring to all substrings
	to the right of it; this process is in essence quadratic.  However,
	only substrings of length at least 'Min_Run_Size' are of interest,
	which gives us the possibility to speed up this process by using
	a hash table.

	For every position p in the text, we construct an index table entry
	forward_reference[p] which gives the next position in the text
	at which a run of Min_Run_Size tokens starts that has the same
	hash code, as calculated by hash1().  If there is no such run,
	the index is 0.

	To construct this array, we use a hash table latest_index[] whose size
	is a prime and which is about the same size as the text array.
	The hash table latest_index[] is set up such that latest_index[i] is the
	index of the latest token with hash_code i, or 0 if there is none.
	See Make_Forward_References().

	The forward references produced this way are not perfect, due to hashing
	coincidences. A second sweep (make_forward_references_perfect()) makes
	them perfect by doing a full comparison on Min_Run_Size tokens.
	For the LaTeX sources of our book Modern Compiler Design, 2nd Ed. this
	reduced the total forward chain length from 103555 to 345, as
	determined by db_forward_reference_check().

	The forward references can be checked with db_forward_reference_check(),
	which also collects statistics.
*/

#include	<stdio.h>
#include	<stdint.h>

#include	"system.par"
#include	"debug.par"
#include	"sim.h"
#include	"text.h"
#include	"Malloc.h"
#include	"any_int.h"
#include	"token.h"
#include	"language.h"
#include	"token.h"
#include	"tokenarray.h"
#include	"options.h"
#include	"hash.h"

							/* MAIN ENTRIES */
static size_t *forward_reference;		/* to be filled by Malloc() */
static size_t n_forward_references;

static void make_forward_references_using_hash(void);
static void make_forward_references_perfect(void);
static void make_chains_circular(void);

#ifdef	DB_FORW_REF
static void db_forward_reference_check(const char *);
static void db_print_forward_references(void);
#endif	/* DB_FORW_REF */

void
Make_Forward_References(void) {
	/*	Constructs the forward references table.
	*/
	n_forward_references = Token_Array_Length();
	forward_reference =
		(size_t *)Calloc(n_forward_references, sizeof (size_t));
	make_forward_references_using_hash();
	make_forward_references_perfect();
	if (is_set_option('a')) {
		make_chains_circular();
	}
#ifdef	DB_FORW_REF_PRINT
	db_print_forward_references();
#endif	/* DB_FORW_REF_PRINT */
}

size_t
Forward_Reference(size_t i, size_t i0) {
	if (i == 0 || i >= n_forward_references) {
		fatal("internal error, bad forward reference");
	}
	size_t new_i = forward_reference[i];
	size_t res = new_i == 0 || new_i == i0 /*circular*/ ? 0 : new_i;
	return res;
}

void
Free_Forward_References(void) {
	Free(forward_reference);
}

							/* HASHING */
static size_t *latest_index;
static size_t latest_index_table_size;

/* The prime numbers of the form 4 * i + 3 for some i, all greater
   than twice the previous one and smaller than 2^40 (for now).
*/
static const uint64_t prime[] = {
#if 0
	3,
	7,
	19,
	43,
	103,
	211,
	431,
	863,
	1747,
	3499,
	7019,
#endif
	14051,
	28111,
	56239,
	112507,
	225023,
	450067,
	900139,
	1800311,
	3600659,
	7201351,
	14402743,
	28805519,
	57611039,
	115222091,
	230444239,
	460888499,
	921777067,
	1843554151,
	UINT64_C (3687108307),
	UINT64_C (7374216631),
	UINT64_C (14748433279),
	UINT64_C (29496866579),
	UINT64_C (58993733159),
	UINT64_C (117987466379),
	UINT64_C (235974932759),
	UINT64_C (471949865531),
	UINT64_C (943899731087)
	/* 2^40= 1099511627776 */
};

static void
init_hash_table(void) {
	int n;

	/* find the ideal hash table size */
	n = 0;
	while (prime[n] < Token_Array_Length()) {
		n++;
		/* this will always terminate, if prime[] is large enough */
	}

	/* see if we can allocate that much space, and if not, step down */
	latest_index = 0;
	while (	/* we have not yet obtained our array */
	        !latest_index
	&&	/* and there is still a (prime) size left to try */
	        n >= 0
	) {
		latest_index_table_size = prime[n];
		latest_index = (size_t *)
			TryCalloc(latest_index_table_size, sizeof (size_t));
		n--;
	}
	if (!latest_index) {
		fatal("out of memory: no room for hash table");
	}
}

static void
make_forward_references_using_hash(void) {
	int n;

	init_hash_table();

	/* Set up the forward references using the latest_index[] hash table. */
	for (n = 0; n < Number_of_Texts; n++) {
		const struct text *txt = &Text[n];
		size_t j;
		uint32_t hash = 0;

#define	Left_Circular_32(i, s)	(((i) << (s)) | ((i) >> (32-(s))))
#define	SHIFT	(5)

		for (j = txt->tx_start;	j < txt->tx_limit; j++) {
			if (	/* we have a complete hash value */
				j - txt->tx_start >= Min_Run_Size
			) {	/* remove the oldest token */
				Token oldest_token =
					Token_Array[j - Min_Run_Size];
				int oldest_shift =
					((Min_Run_Size-1) * SHIFT) % 32;
				hash ^=
				   Left_Circular_32(oldest_token, oldest_shift);
			}
			/* Circular left shift */
			hash = Left_Circular_32(hash, SHIFT);
			/* Add new token */
			hash ^= Token_Array[j];

			/* If have we assembled a complete hash value now,
			   the corresponding run would start at
			   j - (Min_Run_Size - 1). For it to be valid it
			   should start at or after txt->tx_start, so we would
			   like to write the test
			   j - (Min_Run_Size - 1) >= txt->tx_start. However,
			   the type of this computation is size_t, which is
			   unsigned, and j - (Min_Run_Size - 1) may be negative,
			   so we code instead:
			*/
			if (j - txt->tx_start < (Min_Run_Size - 1)) {
				/* no */
				continue;
			}

			/* We now have the complete hash value for a run ending
			   at j and can safely compute j - (Min_Run_Size - 1).
			*/
			size_t run_start = j - (Min_Run_Size - 1);

			/* Can the run be useful? */
			if (!May_Be_Start_Of_Run(Token_Array[run_start]))
				continue;			/* no*/

			/* the hash value is used here for an index */
			size_t h = hash % latest_index_table_size;

			if (latest_index[h]) {
				forward_reference[latest_index[h]] = run_start;
			}
			/*latest_index[h] = j;*/
			latest_index[h] = run_start;
		}
	}

	Free(latest_index);

#ifdef	DB_FORW_REF
	db_forward_reference_check("first hashing");
#endif	/* DB_FORW_REF */
}

static void
make_chains_circular(void) {
	size_t i;

	/* Make the chains circular, by a slightly quadratic algorithm. */
	for (i = 0; i+Min_Run_Size < Token_Array_Length(); i++) {
		if (!forward_reference[i]) continue;
		size_t j = i;
		while (forward_reference[j]) {
			size_t j1 = forward_reference[j];
			if (j1 < j) break;	/* has already been treated */
			j = j1;
		}
		if (forward_reference[j] == 0 && j != i) {
			/* tie it back to the beginning of the chain */
			forward_reference[j] = i;
		}
	}
}

static int
is_eq_min_run(const Token *p, const Token *q) {
	/* a full comparison for the tertiary sweep */
	size_t n;

	for (n = 0; n < Min_Run_Size; n++) {
		if (!Token_EQ(p[n], q[n])) return 0;
	}
	return 1;
}

static void
make_forward_references_perfect(void) {
	size_t i;

	/* Simulate a perfect hash by doing a full comparison
	   over Min_Run_Size, for gathering statistics.
	*/

	for (i = 0; i+Min_Run_Size < Token_Array_Length(); i++) {
		size_t j = i;

		while (	/* there is still a forward reference */
			(j = forward_reference[j])
		&&	/* it does not match over Min_Run_Size */
			!is_eq_min_run(&Token_Array[i], &Token_Array[j])
		) {
			/* continue searching */
		}
		/* short-circuit forward reference to it, or to zero */
		forward_reference[i] = j;
	}
	/* now we have perfect forward references */

#ifdef	DB_FORW_REF
	db_forward_reference_check("full Min_Run_Size comparison");
#endif	/* DB_FORW_REF */
}

#ifdef	DB_FORW_REF

static void
db_print_forward_references(void) {
	/* also determines the lengths of the chains, for statistics */
	size_t n;
	size_t n_frw_chains = 0;
	size_t tot_frwc_len = 0;
	size_t *print_loc_of =
		(size_t *)Calloc(Token_Array_Length(), sizeof (size_t));
	size_t *number_of_chains_of_length =
		(size_t *)Calloc(Token_Array_Length(), sizeof (size_t));

	/* print the references */
	for (n = 1; n < Token_Array_Length(); n++) {
		size_t fw = forward_reference[n];
		if (fw == 0) continue;

		/* we have a chain */
		fprintf(Debug_File, "FWR[%s]:", any_uint2string(n, 0));

		/* is it old? */
		if (print_loc_of[n]) {
			fprintf(Debug_File, " see %s\n",
				any_uint2string(print_loc_of[n], 0));
			continue;
		}

		/* no, we have the beginning of a new chain */
		size_t count = 0;
		do {
			count++;
			fprintf(Debug_File, " %s",
				any_uint2string(fw, 0));
			print_loc_of[fw] = n;
			fw = forward_reference[fw];
		} while(fw && fw != n);	/* continuing and not circular */
		if (fw) {	/* circular */
			fprintf(Debug_File, " C");
			count++;
		}
		n_frw_chains++;
		tot_frwc_len += count;
		number_of_chains_of_length[count]++;
		fprintf(Debug_File, "\n");
	}

	/* print the chain lengths */
	for (n = 1; n < Token_Array_Length(); n++) {
		if (number_of_chains_of_length[n]) {
			fprintf(Debug_File, "length[%d]:\t%d\n",
				n, number_of_chains_of_length[n]);
		}
	}

	fprintf(Debug_File,
		"text length = %s, # forward chains = %s, av. frw chain length = %.2f\n\n",
		any_uint2string(Token_Array_Length(), 0),
		any_uint2string(n_frw_chains, 0),
		(n_frw_chains ? 1.0 * tot_frwc_len / n_frw_chains : 0.0)
	);

	Free(number_of_chains_of_length);
	Free(print_loc_of);
}

static void
db_frw_chain(size_t n, char *crossed_out) {
	if (forward_reference[n] == 0) {
		fprintf(Debug_File,
			">>>> db_frw_chain() forward_reference[n] == 0 <<<<\n"
		);
		return;
	}

	size_t n_entries = 0;
	size_t fw;

	for (fw = n; fw; fw = forward_reference[fw]) {
		if (crossed_out[fw]) {
			fprintf(Debug_File,
				">>>> error: forward references cross <<<<\n"
			);
		}
		n_entries++;
		crossed_out[fw] = 1;
	}
#ifdef	DB_FORW_REF_PRINT
	fprintf(Debug_File, "chain_start = %s, n_entries = %s\n",
		any_uint2string(n, 0), any_uint2string(n_entries, 0));
#endif	/* DB_FORW_REF_PRINT */
}

static void
db_forward_reference_check(const char *msg) {
	/*	Each forward_reference[n] starts in principle a new
		chain, and these chains never touch each other.
		We check this property by marking the positions in each
		chain in an array; if we meet a marked entry while
		following a chain, it must have been on an earlier chain
		and we have an error.
	*/
	size_t n;
	char *crossed_out = (char *)Calloc(Token_Array_Length(), sizeof (char));

	fprintf(Debug_File, "\n\n**** DB_FORWARD_REFERENCES, %s ****\n", msg);
	fprintf(Debug_File, "latest_index_table_size = %s\n",
		any_uint2string(latest_index_table_size, 0));

	if (forward_reference[0]) {
		fprintf(Debug_File,
			">>>> forward_reference[0] is not zero <<<<\n"
		);
	}
	for (n = 1; n < Token_Array_Length(); n++) {
		if (forward_reference[n] && !crossed_out[n]) {
			/* start of a new chain */
			db_frw_chain(n, crossed_out);
		}
	}
#ifdef	DB_FORW_REF_PRINT
	db_print_forward_references();
#endif	/* DB_FORW_REF_PRINT */

	Free(crossed_out);
}

#endif	/* DB_FORW_REF */