Create a type-specific typanalyze routine for tsvector, which collects stats

[PostgreSQL.git] / src / backend / tsearch / ts_typanalyze.c

/*-------------------------------------------------------------------------
 *
 * ts_typanalyze.c
 *        functions for gathering statistics from tsvector columns
 *
 * Portions Copyright (c) 1996-2008, PostgreSQL Global Development Group
 *
 *
 * IDENTIFICATION
 *        $PostgreSQL$
 *
 *-------------------------------------------------------------------------
 */
#include "postgres.h"

#include "access/hash.h"
#include "catalog/pg_operator.h"
#include "commands/vacuum.h"
#include "tsearch/ts_type.h"
#include "utils/builtins.h"
#include "utils/hsearch.h"


/* A hash key for lexemes */
typedef struct
{
        char       *lexeme;                     /* lexeme (not NULL terminated!) */
        int                     length;                 /* its length in bytes */
} LexemeHashKey;

/* A hash table entry for the Lossy Counting algorithm */
typedef struct
{
        LexemeHashKey   key;            /* This is 'e' from the LC algorithm. */
        int                             frequency;      /* This is 'f'. */
        int                             delta;          /* And this is 'delta'. */
} TrackItem;

static void compute_tsvector_stats(VacAttrStats *stats,
                                                                   AnalyzeAttrFetchFunc fetchfunc,
                                                                   int samplerows,
                                                                   double totalrows);
static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current);
static uint32 lexeme_hash(const void *key, Size keysize);
static int lexeme_match(const void *key1, const void *key2, Size keysize);
static int trackitem_compare_desc(const void *e1, const void *e2);


/*
 *      ts_typanalyze -- a custom typanalyze function for tsvector columns
 */
Datum
ts_typanalyze(PG_FUNCTION_ARGS)
{
        VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0);
        Form_pg_attribute attr = stats->attr;

        /* If the attstattarget column is negative, use the default value */
        /* NB: it is okay to scribble on stats->attr since it's a copy */
        if (attr->attstattarget < 0)
                attr->attstattarget = default_statistics_target;

        stats->compute_stats = compute_tsvector_stats;
        /* see comment about the choice of minrows from analyze.c */
        stats->minrows = 300 * attr->attstattarget;

        PG_RETURN_BOOL(true);
}

/*
 *      compute_tsvector_stats() -- compute statistics for a tsvector column
 *
 *      This functions computes statistics that are useful for determining @@
 *      operations' selectivity, along with the fraction of non-null rows and
 *      average width.
 *
 *      Instead of finding the most common values, as we do for most datatypes,
 *      we're looking for the most common lexemes. This is more useful, because
 *      there most probably won't be any two rows with the same tsvector and thus
 *      the notion of a MCV is a bit bogus with this datatype. With a list of the
 *      most common lexemes we can do a better job at figuring out @@ selectivity.
 *
 *      For the same reasons we assume that tsvector columns are unique when
 *      determining the number of distinct values.
 *
 *      The algorithm used is Lossy Counting, as proposed in the paper "Approximate
 *      frequency counts over data streams" by G. S. Manku and R. Motwani, in
 *      Proceedings of the 28th International Conference on Very Large Data Bases,
 *      Hong Kong, China, August 2002, section 4.2. The paper is available at
 *      http://www.vldb.org/conf/2002/S10P03.pdf
 *
 *      The Lossy Counting (aka LC) algorithm goes like this:
 *      Let D be a set of triples (e, f, d), where e is an element value, f is
 *      that element's frequency (occurrence count) and d is the maximum error in
 *      f.  We start with D empty and process the elements in batches of size
 *      w. (The batch size is also known as "bucket size".) Let the current batch
 *      number be b_current, starting with 1. For each element e we either
 *      increment its f count, if it's already in D, or insert a new triple into D
 *      with values (e, 1, b_current - 1). After processing each batch we prune D,
 *      by removing from it all elements with f + d <= b_current. Finally, we
 *      gather elements with largest f.  The LC paper proves error bounds on f
 *      dependent on the batch size w, and shows that the required table size
 *      is no more than a few times w.
 *
 *      We use a hashtable for the D structure and a bucket width of
 *      statistic_target * 100, where 100 is an arbitrarily chosen constant, meant
 *      to approximate the number of lexemes in a single tsvector.
 */
static void
compute_tsvector_stats(VacAttrStats *stats,
                                           AnalyzeAttrFetchFunc fetchfunc,
                                           int samplerows,
                                           double totalrows)
{
        int                             num_mcelem;
        int                             null_cnt = 0;
        double                  total_width = 0;
        /* This is D from the LC algorithm. */
        HTAB                    *lexemes_tab;
        HASHCTL                 hash_ctl;
        HASH_SEQ_STATUS scan_status;
        /* This is the current bucket number from the LC algorithm */
        int                             b_current;
        /* This is 'w' from the LC algorithm */
        int                             bucket_width;
        int vector_no,
                lexeme_no;
        LexemeHashKey   hash_key;
        TrackItem               *item;

        /* We want statistic_target * 100 lexemes in the MCELEM array */
        num_mcelem = stats->attr->attstattarget * 100;

        /*
         * We set bucket width equal to the target number of result lexemes.
         * This is probably about right but perhaps might need to be scaled
         * up or down a bit?
         */
        bucket_width = num_mcelem;

        /*
         * Create the hashtable. It will be in local memory, so we don't need to
         * worry about initial size too much. Also we don't need to pay any
         * attention to locking and memory management.
         */
        MemSet(&hash_ctl, 0, sizeof(hash_ctl));
        hash_ctl.keysize = sizeof(LexemeHashKey);
        hash_ctl.entrysize = sizeof(TrackItem);
        hash_ctl.hash = lexeme_hash;
        hash_ctl.match = lexeme_match;
        hash_ctl.hcxt = CurrentMemoryContext;
        lexemes_tab = hash_create("Analyzed lexemes table",
                                                          bucket_width * 4,
                                                          &hash_ctl,
                                                          HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);

        /* Initialize counters. */
        b_current = 1;
        lexeme_no = 1;

        /* Loop over the tsvectors. */
        for (vector_no = 0; vector_no < samplerows; vector_no++)
        {
                Datum           value;
                bool            isnull;
                TSVector        vector;
                WordEntry       *curentryptr;
                char            *lexemesptr;
                int                     j;

                vacuum_delay_point();

                value = fetchfunc(stats, vector_no, &isnull);

                /*
                 * Check for null/nonnull.
                 */
                if (isnull)
                {
                        null_cnt++;
                        continue;
                }

                /*
                 * Add up widths for average-width calculation.  Since it's a
                 * tsvector, we know it's varlena.  As in the regular
                 * compute_minimal_stats function, we use the toasted width for this
                 * calculation.
                 */
                total_width += VARSIZE_ANY(DatumGetPointer(value));

                /*
                 * Now detoast the tsvector if needed.
                 */
                vector = DatumGetTSVector(value);

                /*
                 * We loop through the lexemes in the tsvector and add them to our
                 * tracking hashtable.  Note: the hashtable entries will point into
                 * the (detoasted) tsvector value, therefore we cannot free that
                 * storage until we're done.
                 */
                lexemesptr = STRPTR(vector);
                curentryptr = ARRPTR(vector);
                for (j = 0; j < vector->size; j++)
                {
                        bool                    found;

                        /* Construct a hash key */
                        hash_key.lexeme = lexemesptr + curentryptr->pos;
                        hash_key.length = curentryptr->len;

                        /* Lookup current lexeme in hashtable, adding it if new */
                        item = (TrackItem *) hash_search(lexemes_tab,
                                                                                         (const void *) &hash_key,
                                                                                         HASH_ENTER, &found);

                        if (found)
                        {
                                /* The lexeme is already on the tracking list */
                                item->frequency++;
                        }
                        else
                        {
                                /* Initialize new tracking list element */
                                item->frequency = 1;
                                item->delta = b_current - 1;
                        }

                        /* We prune the D structure after processing each bucket */
                        if (lexeme_no % bucket_width == 0)
                        {
                                prune_lexemes_hashtable(lexemes_tab, b_current);
                                b_current++;
                        }

                        /* Advance to the next WordEntry in the tsvector */
                        lexeme_no++;
                        curentryptr++;
                }
        }

        /* We can only compute real stats if we found some non-null values. */
        if (null_cnt < samplerows)
        {
                int                     nonnull_cnt = samplerows - null_cnt;
                int                     i;
                TrackItem       **sort_table;
                int                     track_len;

                stats->stats_valid = true;
                /* Do the simple null-frac and average width stats */
                stats->stanullfrac = (double) null_cnt / (double) samplerows;
                stats->stawidth = total_width / (double) nonnull_cnt;

                /* Assume it's a unique column (see notes above) */
                stats->stadistinct = -1.0;

                /*
                 * Determine the top-N lexemes by simply copying pointers from the
                 * hashtable into an array and applying qsort()
                 */
                track_len = hash_get_num_entries(lexemes_tab);

                sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * track_len);

                hash_seq_init(&scan_status, lexemes_tab);
                i = 0;
                while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
                {
                        sort_table[i++] = item;
                }
                Assert(i == track_len);

                qsort(sort_table, track_len, sizeof(TrackItem *),
                          trackitem_compare_desc);

                /* Suppress any single-occurrence items */
                while (track_len > 0)
                {
                        if (sort_table[track_len-1]->frequency > 1)
                                break;
                        track_len--;
                }

                /* Determine the number of most common lexemes to be stored */
                if (num_mcelem > track_len)
                        num_mcelem = track_len;

                /* Generate MCELEM slot entry */
                if (num_mcelem > 0)
                {
                        MemoryContext   old_context;
                        Datum                   *mcelem_values;
                        float4                  *mcelem_freqs;

                        /* Must copy the target values into anl_context */
                        old_context = MemoryContextSwitchTo(stats->anl_context);
                        mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
                        mcelem_freqs = (float4 *) palloc(num_mcelem * sizeof(float4));

                        for (i = 0; i < num_mcelem; i++)
                        {
                                TrackItem *item = sort_table[i];

                                mcelem_values[i] =
                                        PointerGetDatum(cstring_to_text_with_len(item->key.lexeme,
                                                                                                                         item->key.length));
                                mcelem_freqs[i] = (double) item->frequency / (double) nonnull_cnt;
                        }
                        MemoryContextSwitchTo(old_context);

                        stats->stakind[0] = STATISTIC_KIND_MCELEM;
                        stats->staop[0] = TextEqualOperator;
                        stats->stanumbers[0] = mcelem_freqs;
                        stats->numnumbers[0] = num_mcelem;
                        stats->stavalues[0] = mcelem_values;
                        stats->numvalues[0] = num_mcelem;
                        /* We are storing text values */
                        stats->statypid[0] = TEXTOID;
                        stats->statyplen[0] = -1; /* typlen, -1 for varlena */
                        stats->statypbyval[0] = false;
                        stats->statypalign[0] = 'i';
                }
        }
        else
        {
                /* We found only nulls; assume the column is entirely null */
                stats->stats_valid = true;
                stats->stanullfrac = 1.0;
                stats->stawidth = 0;            /* "unknown" */
                stats->stadistinct = 0.0;       /* "unknown" */
        }

        /*
         * We don't need to bother cleaning up any of our temporary palloc's.
         * The hashtable should also go away, as it used a child memory context.
         */
}

/*
 *      A function to prune the D structure from the Lossy Counting algorithm.
 *      Consult compute_tsvector_stats() for wider explanation.
 */
static void
prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current)
{
        HASH_SEQ_STATUS scan_status;
        TrackItem               *item;

        hash_seq_init(&scan_status, lexemes_tab);
        while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
        {
                if (item->frequency + item->delta <= b_current)
                {
                        if (hash_search(lexemes_tab, (const void *) &item->key,
                                                        HASH_REMOVE, NULL) == NULL)
                                elog(ERROR, "hash table corrupted");
                }
        }
}

/*
 * Hash functions for lexemes. They are strings, but not NULL terminated,
 * so we need a special hash function.
 */
static uint32
lexeme_hash(const void *key, Size keysize)
{
        const LexemeHashKey *l = (const LexemeHashKey *) key;

        return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
                                                                   l->length));
}

/*
 *      Matching function for lexemes, to be used in hashtable lookups.
 */
static int
lexeme_match(const void *key1, const void *key2, Size keysize)
{
        const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
        const LexemeHashKey *d2 = (const LexemeHashKey *) key2;

        /* The lexemes need to have the same length, and be memcmp-equal */
        if (d1->length == d2->length &&
                memcmp(d1->lexeme, d2->lexeme, d1->length) == 0)
                return 0;
        else
                return 1;
}

/*
 *      qsort() comparator for TrackItems - LC style (descending sort)
 */
static int
trackitem_compare_desc(const void *e1, const void *e2)
{
        const TrackItem * const *t1 = (const TrackItem * const *) e1;
        const TrackItem * const *t2 = (const TrackItem * const *) e2;

        return (*t2)->frequency - (*t1)->frequency;
}