Optimize V-gram GIN index for "fast scan"

With GIN "fast scan" it doesn't make much sense to exclude entries from GIN
scan just because they are not rare enough.  Thus, we don't estimate selectivity
of V-grams and just extracts all possible V-grams.
Also triconsistent function is implemented for better "fast scan" performance.

Author: Alexander Korotkov

README.md

@@ -98,4 +98,4 @@ installation. It is available from
 [github](https://github.com/akorotkov/vgram)
 under the same license as
 [PostgreSQL](https://www.postgresql.org/about/licence/)
-and supports PostgreSQL 9.2+.
+and supports PostgreSQL 9.4+.

vgram--1.0.sql

@@ -38,9 +38,9 @@ AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE;
 
 CREATE AGGREGATE qgram_stat(text) (
-  SFUNC=qgram_stat_transfn,
-  STYPE=internal,
-  FINALFUNC=qgram_stat_finalfn
+	SFUNC = qgram_stat_transfn,
+	STYPE = internal,
+	FINALFUNC = qgram_stat_finalfn
 );
 
 -- support functions for gin
@@ -59,19 +59,25 @@ RETURNS internal
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
-CREATE FUNCTION vgram_gin_consitent(internal, int2, text, int4, internal, internal, internal, internal)
+CREATE FUNCTION vgram_gin_consistent(internal, int2, text, int4, internal, internal, internal, internal)
 RETURNS bool
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
+CREATE FUNCTION vgram_gin_triconsistent(internal, int2, text, int4, internal, internal, internal)
+RETURNS "char"
+AS 'MODULE_PATHNAME'
+LANGUAGE C IMMUTABLE STRICT;
+
 CREATE OPERATOR CLASS vgram_gin_ops
 FOR TYPE text USING gin
 AS
-        OPERATOR        3       pg_catalog.~~ (text, text),
-        OPERATOR        4       pg_catalog.~~* (text, text),
-        FUNCTION        1       vgram_cmp (text, text),
-        FUNCTION        2       vgram_gin_extract_value (text, internal),
-        FUNCTION        3       vgram_gin_extract_query (text, internal, int2, internal, internal, internal, internal),
-        FUNCTION        4       vgram_gin_consitent (internal, int2, text, int4, internal, internal, internal, internal),
-        STORAGE         text;
+		OPERATOR		3		pg_catalog.~~ (text, text),
+		OPERATOR		4		pg_catalog.~~* (text, text),
+		FUNCTION		1		vgram_cmp (text, text),
+		FUNCTION		2		vgram_gin_extract_value (text, internal),
+		FUNCTION		3		vgram_gin_extract_query (text, internal, int2, internal, internal, internal, internal),
+		FUNCTION		4		vgram_gin_consistent (internal, int2, text, int4, internal, internal, internal, internal),
+		FUNCTION		6		vgram_gin_triconsistent (internal, int2, text, int4, internal, internal, internal),
+		STORAGE			text;
 

vgram_gin.c

@@ -26,9 +26,13 @@ Datum		vgram_gin_extract_value(PG_FUNCTION_ARGS);
 
 PG_FUNCTION_INFO_V1(vgram_gin_extract_value);
 
-Datum		vgram_gin_consitent(PG_FUNCTION_ARGS);
+Datum		vgram_gin_consistent(PG_FUNCTION_ARGS);
 
-PG_FUNCTION_INFO_V1(vgram_gin_consitent);
+PG_FUNCTION_INFO_V1(vgram_gin_consistent);
+
+Datum		vgram_gin_triconsistent(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(vgram_gin_triconsistent);
 
 Datum		vgram_gin_extract_query(PG_FUNCTION_ARGS);
 
@@ -146,7 +150,7 @@ vgram_gin_extract_value(PG_FUNCTION_ARGS)
 }
 
 Datum
-vgram_gin_consitent(PG_FUNCTION_ARGS)
+vgram_gin_consistent(PG_FUNCTION_ARGS)
 {
 	bool	   *check = (bool *) PG_GETARG_POINTER(0);
 	StrategyNumber strategy = PG_GETARG_UINT16(1);
@@ -186,6 +190,44 @@ vgram_gin_consitent(PG_FUNCTION_ARGS)
 	PG_RETURN_BOOL(res);
 }
 
+Datum
+vgram_gin_triconsistent(PG_FUNCTION_ARGS)
+{
+	GinTernaryValue *check = (GinTernaryValue *) PG_GETARG_POINTER(0);
+	StrategyNumber strategy = PG_GETARG_UINT16(1);
+
+	/* text    *query = PG_GETARG_TEXT_P(2); */
+	int32		nkeys = PG_GETARG_INT32(3);
+
+	/* Pointer	  *extra_data = (Pointer *) PG_GETARG_POINTER(4); */
+	GinTernaryValue res = GIN_MAYBE;
+	int32		i;
+
+	switch (strategy)
+	{
+		case ILikeStrategyNumber:
+		case LikeStrategyNumber:
+			/* Check if all extracted trigrams are presented. */
+			for (i = 0; i < nkeys; i++)
+			{
+				if (check[i] == GIN_FALSE)
+				{
+					res = GIN_FALSE;
+					break;
+				}
+			}
+			break;
+		default:
+			elog(ERROR, "unrecognized strategy number: %d", strategy);
+			res = false;		/* keep compiler quiet */
+			break;
+	}
+
+	/* All cases served by this function are inexact */
+	Assert(res != GIN_TRUE);
+	PG_RETURN_GIN_TERNARY_VALUE(res);
+}
+
 Datum
 vgram_gin_extract_query(PG_FUNCTION_ARGS)
 {

vgram_like.c

@@ -156,40 +156,24 @@ get_wildcard_part(const char *str, int lenstr,
 
 typedef struct
 {
-	char	   *vgram;
-	float4		selectivity;
+	char	  **data;
+	int			count;
+	int			allocated;
 }	VGramInfo;
 
 static void
 addVGram(char *vgram, void *userData)
 {
-	float4		selectivity = estimateVGramSelectivilty(vgram);
-	float4		placeSelectivity;
-	int			i,
-				place = -1;
 	VGramInfo  *vgrams = (VGramInfo *) userData;
 
-	for (i = 0; i < OPTIMAL_VGRAM_COUNT; i++)
+	if (vgrams->count >= vgrams->allocated)
 	{
-		if (vgrams[i].vgram == NULL)
-		{
-			vgrams[i].vgram = vgram;
-			vgrams[i].selectivity = selectivity;
-			return;
-		}
-		if ((place < 0 && selectivity < vgrams[i].selectivity) ||
-			(place >= 0 && vgrams[i].selectivity > placeSelectivity))
-		{
-			place = i;
-			placeSelectivity = vgrams[i].selectivity;
-		}
-	}
-	if (place >= 0)
-	{
-		pfree(vgrams[place].vgram);
-		vgrams[place].vgram = vgram;
-		vgrams[place].selectivity = selectivity;
+		vgrams->allocated *= 2;
+		vgrams->data = (char **) repalloc(vgrams->data, sizeof(char *) * vgrams->allocated);
 	}
+
+	vgrams->data[vgrams->count] = vgram;
+	vgrams->count++;
 }
 
 
@@ -204,14 +188,16 @@ extractQueryLike(int32 *nentries, text *pattern)
 				bytelen,
 				charlen,
 				i;
-	VGramInfo	vgrams[OPTIMAL_VGRAM_COUNT];
+	VGramInfo	vgrams;
 	ExtractVGramsInfo userData;
 	Datum	   *entries;
 
 	userData.callback = addVGram;
-	userData.userData = (void *) vgrams;
+	userData.userData = (void *) &vgrams;
 
-	memset(vgrams, 0, sizeof(vgrams));
+	vgrams.count = 0;
+	vgrams.allocated = 16;
+	vgrams.data = (char **) palloc(sizeof(char *) * vgrams.allocated);
 
 	str = (char *) VARDATA_ANY(pattern);
 	len = VARSIZE_ANY_EXHDR(pattern);
@@ -230,20 +216,12 @@ extractQueryLike(int32 *nentries, text *pattern)
 	}
 	pfree(buf);
 
-	*nentries = 0;
-	while (*nentries < OPTIMAL_VGRAM_COUNT && vgrams[*nentries].vgram)
-	{
-		/*
-		 * elog(NOTICE, "%s %f", vgrams[*nentries].vgram,
-		 * vgrams[*nentries].selectivity);
-		 */
-		(*nentries)++;
-	}
+	*nentries = vgrams.count;
 
-	entries = (Datum *) palloc(sizeof(Datum) * (*nentries));
-	for (i = 0; i < *nentries; i++)
+	entries = (Datum *) palloc(sizeof(Datum) * vgrams.count);
+	for (i = 0; i < vgrams.count; i++)
 	{
-		entries[i] = PointerGetDatum(cstring_to_text(vgrams[i].vgram));
+		entries[i] = PointerGetDatum(cstring_to_text(vgrams.data[i]));
 	}
 	return entries;
 }