config: make conv dir configurable

[rb-79.git] / sanitize-comment.c

/*
 * Copyright (c) 2017-2018, De Rais <derais@cock.li>
 *
 * Permission to use, copy, modify, and/or distribute this software for
 * any purpose with or without fee is hereby granted, provided that the
 * above copyright notice and this permission notice appear in all
 * copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
 */
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <wchar.h>

#define PCRE2_CODE_UNIT_WIDTH 8
#include <pcre2.h>

#include "macros.h"
#include "rb79.h"
#include "unicode-transforms.h"

/*
 * We need a way to get codepoints out of UTF-8 strings and if
 * wchar_t stored codepoint values, that would be great. That's
 * __STDC_ISO_10646__, though. You can remove this check and cross
 * your fingers, since rb79 will do a quick check on startup, but
 * please check why the C implementation doesn't define
 * __STDC_ISO_10646__ first.
 */
#ifndef __STDC_ISO_10646__
#error We really want __STD_ISO_10646__
#endif

/*
 * A wordfilter consists of a pcre2 regex and a replacement string
 */
struct wordfilter {
        /* */
        pcre2_code *code;
        const char *replacement;
        size_t replacement_len;
};

/* These are constructed in setup_sanitize_comment() */
static struct wordfilter *wordfilters;
static size_t wordfilters_num;

/* Special matcher for quoting, newlines, linkifying, etc. */
static pcre2_code *format_replacements;

/*
 * Comparison function for struct translate.
 *
 * Preconditions:
 *
 *  - *key_v is a wchar_t.
 *
 *  - *tr_v is a struct translate object.
 *
 * Postconditions:
 *
 *  - Returns -1 (0) [1] if *key_v is less than (equal to) [greater
 *    than] *tr_v's starting range.
 */
static int match_translate(const void *key_v, const void *tr_v)
{
        const wchar_t *key = key_v;
        const struct translate *tr = tr_v;

        if (*key < tr->from_s) {
                return -1;
        } else if (*key > tr->from_t) {
                return 1;
        }

        return 0;
}

/*
 * Add a UTF-8 sequence str onto *buf
 *
 * Preconditions:
 *
 *  - *buf is memory of length *sz, and up to *idx is a valid UTF-8
 *    string.
 *
 *  - str is a valid ASCII (not just UTF-8) string of length str_len.
 *
 * Postconditions (success):
 *
 *  - *buf is memory of length *sz, and up to *idx is a valid UTF-8
 *    string.
 *
 *  - The contents of str have been appended to *buf (and *idx
 *    includes this).
 */
static int append_str(char **buf, size_t *idx, size_t *sz, const char *str,
                      size_t str_len)
{
        if (str_len + *idx >= *sz) {
                void *newmem = 0;
                size_t new_sz = str_len + *idx + (1 << 9);

                if (str_len + *idx < str_len ||
                    str_len + *idx + (1 << 9) < str_len + *idx) {
                        ERROR_MESSAGE("overflow (str_len = %zu, *idx = %zu)",
                                      str_len, *idx);

                        return -1;
                }

                if (!(newmem = realloc(*buf, new_sz))) {
                        PERROR_MESSAGE("realloc");

                        return -1;
                }

                *buf = newmem;
                *sz = new_sz;
        }

        strncpy(*buf + *idx, str, str_len);
        *(*buf + *idx + str_len) = '\0';
        *idx += str_len;

        return 0;
}

/* Dummy function for when I can't be bothered to strlen(). */
static int append_const_str(char **buf, size_t *idx, size_t *len, const
                            char *str)
{
        return append_str(buf, idx, len, str, strlen(str));
}

/*
 * Add a single character onto *buf
 *
 * Preconditions:
 *
 *  - *buf is memory of length *len, and up to *idx is a valid UTF-8
 *    string.
 *
 *  - c is an ASCII character.
 *
 * Postconditions (success):
 *
 *  - *buf is memory of length *len, and up to *idx is a valid UTF-8
 *    string.
 *
 *  - c has been appended to *buf (and *idx includes this).
 */
static int append_char(char **buf, size_t *idx, size_t *len, char c)
{
        if (1 + *idx >= *len) {
                void *newmem = 0;
                size_t new_len = 1 + *idx + (1 << 9);

                if (*idx + 1 < *idx ||
                    *idx + 1 + (1 << 9) < *idx + 1) {
                        ERROR_MESSAGE("overflow (*idx = %zu)", *idx);

                        return -1;
                }

                if (!(newmem = realloc(*buf, new_len))) {
                        PERROR_MESSAGE("realloc");

                        return -1;
                }

                *buf = newmem;
                *len = new_len;
        }

        *(*buf + *idx) = c;
        *(*buf + *idx + 1) = '\0';
        *idx += 1;

        return 0;
}

/*
 * Add a Unicode codepoint onto *buf
 *
 * Preconditions:
 *
 *  - *buf is memory of length *sz, and up to *idx is a valid UTF-8
 *    string.
 *
 *  - wchar_t is a valid Unicode codepoint.
 *
 * Postconditions (success):
 *
 *  - *buf is memory of length *sz, and up to *idx is a valid UTF-8
 *    string.
 *
 *  - An HTML-escaped sequence like &#123; has been appended to
 *    *buf (and *idx includes this).
 */
static int append_wchar_escaped(char **buf, size_t *idx, size_t *sz, wchar_t wc)
{
        size_t l = snprintf(0, 0, "&#%ld;", (long) wc);

        if (l + *idx >= *sz) {
                void *newmem = 0;
                size_t new_sz = l + *idx + (1 << 9);

                if (*idx + l < *idx ||
                    *idx + l + (1 << 9) < *idx + l) {
                        ERROR_MESSAGE("overflow (*idx = %zu, l = %zu)", *idx,
                                      l);

                        return -1;
                }

                if (!(newmem = realloc(*buf, new_sz))) {
                        PERROR_MESSAGE("realloc");

                        return -1;
                }

                *buf = newmem;
                *sz = new_sz;
        }

        sprintf(*buf + *idx, "&#%ld;", (long) wc);
        *idx += l;

        return 0;
}

/*
 * Ensure that (*map)[j] = k, fixing up length as appropriate.
 *
 * Preconditions
 *
 *  - *map is memory of length len.
 *
 * Postconditions (success):
 *
 *  - *map is memory of length len.
 *
 *  - (*map)[j] = k.
 */
static int set_position_mapping(size_t **map, size_t *len, size_t j, size_t k)
{
        if (j + 1 >= *len) {
                void *newmem = 0;

                if (j + 2 < j ||
                    ((j + 2) * sizeof **map) / (j + 2) != sizeof **map) {
                        ERROR_MESSAGE("overflow (j = %zu)", j);

                        return -1;
                }

                if (!(newmem = realloc(*map, (j + 2) * sizeof **map))) {
                        PERROR_MESSAGE("realloc");

                        return -1;
                }

                *map = newmem;

                for (size_t l = *len; l < j + 2; ++l) {
                        (*map)[l] = ((size_t) -1);
                }

                *len = j + 2;
        }

        (*map)[j] = k;

        return 0;
}

/*
 * HTML-escape in to *out.
 *
 * Preconditions
 *
 *  - in is memory of at least length in_len, valid UTF-8
 *    text.
 *
 *  - *out is memory of at least length *out_len (if *out_len = 0,
 *    *out may be 0), valid UTF-8 text.
 *
 *  - Overwriting *out and *out_len  shall not cause a memory leak.
 *
 *  - out, out_len, and out_idx are not 0.
 *
 * Postconditions (success):
 *
 *  - *out is memory of at least length *out_len, valid UTF-8 text.
 *
 *  - A stretch of HTML-escaped ASCII text representing in has been
 *    added to *out at the position that was *out_idx.
 *
 *  - *out_idx has been updated to point to the end of this stretch.
 *
 *  - If necessary, *out_len has been updated.
 */
static int to_html(const char *in, const size_t in_len, size_t in_idx,
                   char **out, size_t *out_len, size_t *out_idx)
{
        int ret = -1;
        wchar_t wc = 0;
        int mbret = 0;
        size_t out_sz = 0;
        size_t initial_out_idx = *out_idx;

        if (!*out) {
                if (!(*out = malloc(1))) {
                        PERROR_MESSAGE("malloc");
                        goto done;
                }

                out_sz = 1;
                *out_len = 0;
                (*out)[0] = '\0';
        }

        /*
         * XXX: If you make this multithreaded, be sure to use
         * mbrtowc(3) here!
         */
        while (in_idx < in_len &&
               in[in_idx]) {
                /* Extract next character */
                mbret = mbtowc(&wc, in + in_idx, in_len - in_idx);

                if (mbret == -1) {
                        PERROR_MESSAGE("mbtowc");
                        goto done;
                }

                if (wc == L'&') {
                        ret = append_str(out, out_idx, &out_sz, "&amp;", 5);
                } else if (wc == L'"') {
                        ret = append_str(out, out_idx, &out_sz, "&quot;", 6);
                } else if (wc == L'\'') {
                        ret = append_str(out, out_idx, &out_sz, "&apos;", 6);
                } else if (wc == L'<') {
                        ret = append_str(out, out_idx, &out_sz, "&lt;", 4);
                } else if (wc == L'>') {
                        ret = append_str(out, out_idx, &out_sz, "&gt;", 4);
                } else if (mbret == 1 &&
                           in[in_idx] >= ' ' &&
                           in[in_idx] <= '~') {
                        ret = append_char(out, out_idx, &out_sz, in[in_idx]);
                } else if (mbret == 1 &&
                           in[in_idx] == '\r') {
                        ret = 0;
                } else if (mbret == 1 &&
                           in[in_idx] == '\n') {
                        ret = append_char(out, out_idx, &out_sz, in[in_idx]);
                } else {
                        ret = append_wchar_escaped(out, out_idx, &out_sz, wc);
                }

                in_idx += mbret;

                if (ret < 0) {
                        goto done;
                }
        }

        *out_len = *out_len + (*out_idx - initial_out_idx);
        ret = 0;
done:

        return ret;
}

/*
 * From in construct *out, which is a codepoint-for-codepoint
 * translation following the rules of unicode-transforms.h. The
 * result is that *out can be matched with normal regex, even if
 * in contains obfuscatory Unicode bullshit.
 *
 * Preconditions
 *
 *  - setup_sanitize_comment() has been invoked more recently than
 *    clean_sanitize_comment().
 *
 *  - in is memory of at least length in_len, valid UTF-8 text.
 *
 *  - Overwriting *out and *out_position_map shall not cause a
 *    memory leak.
 *
 *  - out, out_len, out_position_map, and out_position_map_len are
 *    not 0.
 *
 * Postconditions (success):
 *
 *  - *out is valid, UTF-8 text of length *out_len.
 *
 *  - For every j in [0, *out_len) such that (*out)[j] starts a
 *    codepoint, in[*(position_map)[j]] is the start of the
 *    corresponding codepoint.
 *
 *  - (*position_map)[*out_len] = in_len.
 */
static int to_scannable(const char *in, size_t in_len, char **out,
                        size_t *out_len, size_t **out_position_map,
                        size_t *out_position_map_len)
{
        int ret = -1;
        wchar_t wc = 0;
        size_t in_idx = 0;
        size_t out_idx = 0;
        int mbret = 0;
        struct translate *tr = 0;
        size_t out_sz = 0;

        if (!*out) {
                if (!(*out = malloc(1))) {
                        PERROR_MESSAGE("malloc");
                        goto done;
                }

                out_sz = 1;
                *out_len = 0;
                (*out)[0] = '\0';
        }

        /*
         * Position_map is here to make wordfiltering work. Suppose in is
         *
         *     Ｉ  ｔｈｉｎｋ  Ｎｉｎａ  Ｐｕｒｐｌｅｔｏｎ  ｄｉｄ
         *     ｎｏｔｈｉｎｇ  ｗｒｏｎｇ
         *
         * and a wordfilter /Nina Purpleton/i -> "worst girl" is
         * in effect. Then *out will be
         *
         *      I think Nina Purpleton did nothing wrong
         *
         * The message should, of course, be filtered to
         *
         *     Ｉ  ｔｈｉｎｋ worst girl ｄｉｄ ｎｏｔｈｉｎｇ
         *     ｗｒｏｎｇ
         *
         * In order to do that, it would be necessary to have a map
         * from in to *out on the byte level, since the wordfilter
         * will only be run against *out.
         *
         * position_map[j] = k means that out[j] and in[k] mean the
         * same thing.
         */
        while (in_idx < in_len) {
                mbret = mbtowc(&wc, in + in_idx, in_len - in_idx);

                if (mbret == -1) {
                        PERROR_MESSAGE("mbtowc");
                        goto done;
                }

                /* We pre-suppose that the insert will go as planned */
                if (set_position_mapping(out_position_map, out_position_map_len,
                                         out_idx, in_idx) < 0) {
                        goto done;
                }

                if (mbret == 1 &&
                    in[in_idx] >= ' ' &&
                    in[in_idx] <= '~') {
                        if (append_str(out, &out_idx, &out_sz, in + in_idx, 1) <
                            0) {
                                goto done;
                        }
                } else {
                        if ((tr = bsearch(&wc, translates, NUM_OF(translates),
                                          sizeof *translates,
                                          match_translate))) {
                                if (append_str(out, &out_idx, &out_sz, tr->to,
                                               strlen(tr->to)) < 0) {
                                        goto done;
                                }
                        } else {
                                if (append_str(out, &out_idx, &out_sz, in +
                                               in_idx, mbret) < 0) {
                                        goto done;
                                }
                        }
                }

                in_idx += mbret;
        }

        if (set_position_mapping(out_position_map, out_position_map_len,
                                 out_idx, in_len) < 0) {
                goto done;
        }

        (*out)[out_idx] = '\0';
        *out_len = out_idx;
        ret = 0;
done:

        return ret;
}

/*
 * Read through raw and scannable, checking all wordfilters in
 * scannable. Where a match is detected, the corresponding postion
 * (via position_map) in raw is replaced by the replacement specified
 * by the matching wordfilter.
 *
 * Preconditions
 *
 *  - setup_sanitize_comment() has been invoked more recently than
 *    clean_sanitize_comment().
 *
 *  - raw is memory of length at least raw_len, valid UTF-8 text.
 *
 *  - scannable is memory of length at least scannable_len.
 *
 *  - For any j in [0, scannable_len), position_map[j] is a valid
 *    index into raw, or is (size_t) -1.
 *
 *  - position_map[scannable_len] = raw_len.
 *
 *  - For any j in [0, scannable_len) such that k = position_map[j]
 *    is not (size_t) -1, scannable[j] and raw[k] are conceptually
 *    the same for wordfiltering.
 *
 *  - Overwriting *out shall not cause a memory leak.
 *
 *  - out and out_len are not 0.
 *
 * Postconditions (success):
 *
 *  - *out is valid, UTF-8 text of length *out_len such that all
 *    non ASCII codepoints (and '<', '>', '&', '"', ''') are
 *    HTML-escaped.
 *
 *  - *out represents raw, except in those sections of scannable
 *    where a wordfilter matched.
 */
static int wordfilter_to_html(const char *raw, const size_t raw_len, const
                              char *scannable, const size_t scannable_len,
                              size_t *position_map, char **out,
                              size_t *out_len)
{
        int ret = -1;

        /* These hold the match locations from pcre2 */
        uint32_t *ov_counts = 0;
        PCRE2_SIZE **ov_ps = 0;
        int *num_matches = 0;
        pcre2_match_data **match_data = 0;
        size_t raw_idx = 0;
        size_t scannable_idx = 0;
        size_t out_idx = 0;
        size_t best_match_pos = 0;
        size_t best_match_idx = 0;
        size_t l = 0;
        size_t mbret = 0;

        if (!(ov_counts = calloc(wordfilters_num, sizeof *ov_counts))) {
                PERROR_MESSAGE("calloc");
                goto done;
        }

        if (!(ov_ps = calloc(wordfilters_num, sizeof *ov_ps))) {
                PERROR_MESSAGE("calloc");
                goto done;
        }

        if (!(num_matches = calloc(wordfilters_num, sizeof *num_matches))) {
                PERROR_MESSAGE("calloc");
                goto done;
        }

        if (!(match_data = calloc(wordfilters_num, sizeof *match_data))) {
                PERROR_MESSAGE("calloc");
                goto done;
        }

        /* First scan, before the loop */
        for (size_t j = 0; j < wordfilters_num; ++j) {
                if (!(match_data[j] = pcre2_match_data_create_from_pattern(
                              wordfilters[j].code, 0))) {
                        PERROR_MESSAGE("pcre2_match_data_create_from_pattern");
                        goto done;
                }

                num_matches[j] = pcre2_match(wordfilters[j].code,
                                             (PCRE2_SPTR) scannable,
                                             scannable_len, scannable_idx, 0,
                                             match_data[j], 0);
        }

handle_next_match:
        best_match_pos = (size_t) -1;
        best_match_idx = (size_t) -1;

        /* We've run pcre2_match() on everything. Find the soonest match */
        for (size_t j = 0; j < wordfilters_num; ++j) {
                if (!num_matches[j]) {
                        continue;
                }

                ov_ps[j] = pcre2_get_ovector_pointer(match_data[j]);

                if (ov_ps[j][0] >= scannable_idx &&
                    ov_ps[j][0] < best_match_pos) {
                        best_match_pos = ov_ps[j][0];
                        best_match_idx = j;
                }
        }

        if (best_match_idx == (size_t) -1) {
                /* No matches. Turn the rest to html boring-like */
                ret = to_html(raw, raw_len, raw_idx, out, out_len, &out_idx);
                goto done;
        }

        /* Figure out where in raw this match starts */
        l = best_match_pos;

        while (l != (size_t) -1 &&
               position_map[l] == (size_t) -1) {
                l--;
        }

        if (l == (size_t) -1) {
                ERROR_MESSAGE("Impossible condition in "
                              "wordfilter_to_html: raw=\"%s\", best_match_pos = %zu",
                              raw,
                              best_match_pos);
                goto done;
        }

        /*
         * Now position_map[l] points to the first character in raw
         * that should be replaced. Fill up to that point.
         */
        if (position_map[l] &&
            position_map[l] > raw_idx) {
                if (to_html(raw, position_map[l], raw_idx, out, out_len,
                            &out_idx) < 0) {
                        goto done;
                }
        }

        /* Put the substituted text in */
        if (to_html(wordfilters[best_match_idx].replacement,
                    wordfilters[best_match_idx].replacement_len, 0, out,
                    out_len,
                    &out_idx) < 0) {
                goto done;
        }

        /*
         * Figure out where we should advance to in inputs. Naively,
         * we want to set scannable_idx to ov_ps[best_match_idx][1]
         * (the first character in scannable beyond the match).
         * However, we have to consider the case of
         *
         *      foo！！！bar
         *
         * where "foo" -> "baz" is the only transformation. Since
         * some characters, like "！", are completely ignored by
         * the scannable transformation, the naive method would
         * start our scanning at the "b", skipping information.
         *
         * So, instead, we carefully find the last character in
         * "foo", then jump one past it. This (unfortunately)
         * requires a bit more manual fiddling with wide character
         * conversions.
         *
         */
        if (ov_ps[best_match_idx][1] <= scannable_idx) {
                /*
                 * This should never happen, but let's make sure
                 * we always keep advancing.
                 */
                scannable_idx++;
        } else {
                scannable_idx = ov_ps[best_match_idx][1] - 1;
        }

        l = scannable_idx;

        while (position_map[l] == (size_t) -1) {
                l--;
        }

        raw_idx = position_map[l];

        /* This is the "jump one past it" part */
        scannable_idx++;
        errno = 0;
        mbret = mbrlen(raw + raw_idx, MB_CUR_MAX, 0);

        switch (mbret) {
        case (size_t) -2:
        case (size_t) -1:
                PERROR_MESSAGE("mbrlen");
                goto done;
        default:
                raw_idx += mbret;
        }

        /*
         * Now re-check all our matches and figure out which ones
         * need to be updated
         */
        for (size_t j = 0; j < wordfilters_num; ++j) {
                if (!num_matches[j] ||
                    ov_ps[j][0] >= scannable_idx) {
                        continue;
                }

                num_matches[j] = pcre2_match(wordfilters[j].code,
                                             (PCRE2_SPTR) scannable,
                                             scannable_len, scannable_idx, 0,
                                             match_data[j], 0);
        }

        goto handle_next_match;
done:

        for (size_t j = 0; j < wordfilters_num; ++j) {
                pcre2_match_data_free(match_data[j]);
                match_data[j] = 0;
        }

        free(match_data);
        free(num_matches);
        free(ov_counts);
        free(ov_ps);

        return ret;
}

/*
 * Read through in. Each time a match for format_replacements is
 * found (something like a newline or a quote) is found, replace
 * it with some HTML markup. The result is placed in out.
 *
 * Preconditions:
 *
 *  - setup_sanitize_comment() has been invoked more recently than
 *    clean_sanitize_comment().
 *
 *  - in is memory of length at least in_len, valid UTF-8 text.
 *
 *  - Overwriting *out shall not cause a memory leak.
 *
 *  - out and out_len are not 0.
 *
 * Postconditions (success):
 *
 *  - *out is valid, UTF-8 text of length *out_len with sane HTML
 *    markup (and HTML escaped), suitable for outputting into an
 *    HTML file.
 */
static int insert_html_tags(const char *in, size_t in_len, const char *board,
                            char **out, size_t *out_len)
{
        int ret = -1;
        size_t in_idx = 0;
        size_t match_pos = 0;
        size_t after_match_pos = 0;
        size_t out_idx = 0;
        pcre2_match_data *match_data = 0;
        int nret = 0;
        PCRE2_UCHAR *tmp_1 = 0;
        PCRE2_SIZE tmp_1_len = 0;
        PCRE2_UCHAR *tmp_2 = 0;
        PCRE2_SIZE tmp_2_len = 0;
        PCRE2_UCHAR *tmp_3 = 0;
        PCRE2_SIZE tmp_3_len = 0;
        uint_fast8_t last_was_newline = 1;
        char *link_target = 0;
        size_t link_target_len = 0;

        if (!(match_data = pcre2_match_data_create_from_pattern(
                      format_replacements, 0))) {
                PERROR_MESSAGE("pcre2_match_data_create_from_pattern");
                goto done;
        }

find_next_bit:

        if (in_idx >= in_len) {
                goto success;
        }

        nret = pcre2_match(format_replacements, (PCRE2_SPTR) in, in_len, in_idx,
                           0, match_data, 0);

        if (nret == PCRE2_ERROR_NOMATCH) {
                ret = append_str(out, &out_idx, out_len, in + in_idx, in_len -
                                 in_idx);
                goto done;
        }

        if (nret < 0) {
                PCRE2_UCHAR8 err_buf[120];

                pcre2_get_error_message(nret, err_buf, 120);
                ERROR_MESSAGE("pcre2_match: error while matching \"%.*s\": %s"
                              " (PCRE2 %d)", (int) (in_len - in_idx), in +
                              in_idx, err_buf,
                              nret);
                goto done;
        }

        pcre2_substring_free(tmp_1);
        pcre2_substring_free(tmp_2);
        pcre2_substring_free(tmp_3);
        free(link_target);
        tmp_1 = 0;
        tmp_2 = 0;
        tmp_3 = 0;
        link_target = 0;

        /* We have match, stuff everything up to it in *out */
        match_pos = pcre2_get_ovector_pointer(match_data)[0];
        after_match_pos = pcre2_get_ovector_pointer(match_data)[1];

        if (match_pos > in_idx) {
                if (append_str(out, &out_idx, out_len, in + in_idx, match_pos -
                               in_idx) < 0) {
                        goto done;
                }

                last_was_newline = 0;
                in_idx = match_pos;
        }

        /* Figure out what type of match. */
        if (!pcre2_substring_get_byname(match_data, (PCRE2_SPTR) "newline",
                                        &tmp_1, &tmp_1_len)) {
                if (last_was_newline) {
                        if (append_const_str(out, &out_idx, out_len,
                                             "&nbsp;<br />") < 0) {
                                goto done;
                        }
                } else {
                        if (append_const_str(out, &out_idx, out_len, "<br />") <
                            0) {
                                goto done;
                        }
                }

                last_was_newline = 1;
                in_idx = after_match_pos;
                goto find_next_bit;
        }

        last_was_newline = 0;

        if (!pcre2_substring_get_byname(match_data, (PCRE2_SPTR) "quote",
                                        &tmp_1, &tmp_1_len)) {
                if (append_const_str(out, &out_idx, out_len,
                                     "<span class=\"quote\">") < 0) {
                        goto done;
                }

                if (append_str(out, &out_idx, out_len, (const char *) tmp_1,
                               (size_t) tmp_1_len) < 0) {
                        goto done;
                }

                if (append_const_str(out, &out_idx, out_len, "</span>") < 0) {
                        goto done;
                }

                in_idx = after_match_pos;
                goto find_next_bit;
        }

        if (!pcre2_substring_get_byname(match_data,
                                        (PCRE2_SPTR) "intra_postlink", &tmp_1,
                                        &tmp_1_len)) {
                if (pcre2_substring_get_byname(match_data, (PCRE2_SPTR) "a_num",
                                               &tmp_2, &tmp_2_len)) {
                        goto problem_with_match;
                }

                int found = 0;

                if (db_construct_post_link(board, strlen(board), (const
                                                                  char *) tmp_2,
                                           tmp_2_len, &found, &link_target,
                                           &link_target_len) < 0) {
                        goto done;
                }

                if (!found) {
                        if (append_str(out, &out_idx, out_len, in + match_pos,
                                       after_match_pos - match_pos) < 0) {
                                goto done;
                        }

                        in_idx = after_match_pos;
                        goto find_next_bit;
                }

                if (append_const_str(out, &out_idx, out_len, "<a href=\"") <
                    0) {
                        goto done;
                }

                if (append_str(out, &out_idx, out_len, link_target,
                               link_target_len) < 0) {
                        goto done;
                }

                if (append_const_str(out, &out_idx, out_len, "\">") < 0) {
                        goto done;
                }

                if (append_str(out, &out_idx, out_len, (const char *) tmp_1,
                               (size_t) tmp_1_len) < 0) {
                        goto done;
                }

                if (append_const_str(out, &out_idx, out_len, "</a>") < 0) {
                        goto done;
                }

                in_idx = after_match_pos;
                goto find_next_bit;
        }

        if (!pcre2_substring_get_byname(match_data,
                                        (PCRE2_SPTR) "inter_postlink", &tmp_1,
                                        &tmp_1_len)) {
                if (pcre2_substring_get_byname(match_data, (PCRE2_SPTR) "e_num",
                                               &tmp_2, &tmp_2_len)) {
                        goto problem_with_match;
                }

                if (pcre2_substring_get_byname(match_data,
                                               (PCRE2_SPTR) "e_board", &tmp_3,
                                               &tmp_3_len)) {
                        goto problem_with_match;
                }

                int found = 0;

                if (db_construct_post_link((const char *) tmp_3, tmp_3_len,
                                           (const char *) tmp_2, tmp_2_len,
                                           &found, &link_target,
                                           &link_target_len) < 0) {
                        goto done;
                }

                if (!found) {
                        if (append_str(out, &out_idx, out_len, in + match_pos,
                                       after_match_pos - match_pos) < 0) {
                                goto done;
                        }

                        in_idx = after_match_pos;
                        goto find_next_bit;
                }

                if (append_const_str(out, &out_idx, out_len, "<a href=\"") <
                    0) {
                        goto done;
                }

                if (append_str(out, &out_idx, out_len, link_target,
                               link_target_len) < 0) {
                        goto done;
                }

                if (append_const_str(out, &out_idx, out_len, "\">") < 0) {
                        goto done;
                }

                if (append_str(out, &out_idx, out_len, (const char *) tmp_1,
                               (size_t) tmp_1_len) < 0) {
                        goto done;
                }

                if (append_const_str(out, &out_idx, out_len, "</a>") < 0) {
                        goto done;
                }

                in_idx = after_match_pos;
                goto find_next_bit;
        }

problem_with_match:

        /* There was some kind of match, but it went wrong. */
        in_idx++;
        goto find_next_bit;
success:
        ret = 0;
done:
        *out_len = out_idx;
        pcre2_substring_free(tmp_1);
        pcre2_substring_free(tmp_2);
        pcre2_substring_free(tmp_3);
        pcre2_match_data_free(match_data);

        return ret;
}

/*
 * Make sure that the contents of *pc are ready for safe injection
 * into the board, including HTML escaping, wordfiltering, general
 * formatting, and adding links.
 *
 * Preconditions
 *
 *  - setup_sanitize_comment() has been invoked more recently than
 *    clean_sanitize_comment().
 *
 *  - *pc has been filled out (fields like action, board, etc. have
 *    been populated) from the POST data.
 *
 * Postconditions (success):
 *
 *  - The prepared_XYZ fields of *pc have been filled out, and each
 *    is valid ASCII text, with Unicode codepoints.
 */
int st_sanitize_text(struct post_cmd *pc, int *our_fault)
{
        int ret = -1;
        size_t out_idx = 0;
        char *html_escaped_comment = 0;
        size_t html_escaped_comment_len = 0;

        /* Flush out lurking double-free bugs */
        free(pc->prepared.name);
        pc->prepared.name = 0;
        pc->prepared.name_len = 0;
        free(pc->prepared.email);
        pc->prepared.email = 0;
        pc->prepared.email_len = 0;
        free(pc->prepared.subject);
        pc->prepared.subject = 0;
        pc->prepared.subject_len = 0;
        free(pc->prepared.comment);
        pc->prepared.comment = 0;
        pc->prepared.comment_len = 0;
        free(pc->prepared.file_name);
        pc->prepared.file_name = 0;
        pc->prepared.file_name_len = 0;
        free(pc->scannable_comment);
        pc->scannable_comment = 0;
        pc->scannable_comment_len = 0;
        free(pc->position_map);
        pc->position_map = 0;
        pc->position_map_len = 0;
        out_idx = 0;

        if (!pc->raw.name_len) {
                free(pc->raw.name);

                if (!(pc->raw.name = strdup("Anonymous"))) {
                        PERROR_MESSAGE("strdup");
                        goto done;
                }

                pc->raw.name_len = strlen(pc->raw.name);
        }

        if (pc->raw.name_len) {
                if (to_html(pc->raw.name, pc->raw.name_len, 0,
                            &pc->prepared.name, &pc->prepared.name_len,
                            &out_idx) < 0) {
                        *our_fault = 1;
                        goto done;
                }
        }

        out_idx = 0;

        if (pc->raw.email_len) {
                if (to_html(pc->raw.email, pc->raw.email_len, 0,
                            &pc->prepared.email, &pc->prepared.email_len,
                            &out_idx) < 0) {
                        *our_fault = 1;
                        goto done;
                }
        }

        out_idx = 0;

        if (pc->raw.tripcode_len) {
                if (to_html(pc->raw.tripcode, pc->raw.tripcode_len, 0,
                            &pc->prepared.tripcode, &pc->prepared.tripcode_len,
                            &out_idx) <
                    0) {
                        *our_fault = 1;
                        goto done;
                }
        }

        out_idx = 0;

        if (pc->raw.subject_len) {
                if (to_html(pc->raw.subject, pc->raw.subject_len, 0,
                            &pc->prepared.subject, &pc->prepared.subject_len,
                            &out_idx) <
                    0) {
                        *our_fault = 1;
                        goto done;
                }
        }

        out_idx = 0;

        if (pc->raw.file_name_len) {
                if (to_html(pc->raw.file_name, pc->raw.file_name_len, 0,
                            &pc->prepared.file_name,
                            &pc->prepared.file_name_len,
                            &out_idx) < 0) {
                        *our_fault = 1;
                        goto done;
                }
        }

        if (to_scannable(pc->raw.comment, pc->raw.comment_len,
                         &pc->scannable_comment, &pc->scannable_comment_len,
                         &pc->position_map,
                         &pc->position_map_len)) {
                *our_fault = 1;
                goto done;
        }

        /*
         * Now we do the fancy thing. Match scannable, build prepared
         * out of that.
         */
        if (wordfilter_to_html(pc->raw.comment, pc->raw.comment_len,
                               pc->scannable_comment, pc->scannable_comment_len,
                               pc->position_map,
                               &html_escaped_comment,
                               &html_escaped_comment_len) < 0) {
                *our_fault = 1;
                goto done;
        }

        /*
         * Everything's in &#123; form, but now take care of >>123,
         * <br />, etc.
         */
        if (insert_html_tags(html_escaped_comment, html_escaped_comment_len,
                             pc->raw.board, &pc->prepared.comment,
                             &pc->prepared.comment_len) < 0) {
                *our_fault = 1;
                goto done;
        }

        ret = 0;
done:
        free(html_escaped_comment);

        return ret;
}

/*
 * Initialize any static elements needed for this file.
 *
 * Preconditions:
 *
 *  - setup_sanitize_comment() was not invoked more recently than
 *    clean_sanitize_comment().
 *
 * Postconditions (success):
 *
 *  - Any other function in this file may be safely called.
 */
int setup_sanitize_comment(const struct configuration *conf)
{
        /*
         * Check that the locale/libc/whatever is set up so that
         * UTF-8 handling can work.
         */
        int ret = -1;
        const char *raw =
                "<script>alert(1)</script> , \U0001d511\U0001d526\U0001d52b"
                "\U0001d51e\u3000\U0001d513\U0001d532\U0001d52f\U0001d52d"
                "\U0001d529\U0001d522\U0001d531\U0001d52c\U0001d52b & "
                "\u2468\u0294!\u0ce2!!";
        const char *correct_html =
                "&lt;script&gt;alert(1)&lt;/script&gt; , &#120081;&#120102;"
                "&#120107;&#120094;&#12288;&#120083;&#120114;&#120111;"
                "&#120109;&#120105;&#120098;&#120113;&#120108;&#120107; &amp;"
                " &#9320;&#660;!&#3298;!!";
        const char *correct_scannable =
                "<script>alert(1)</script> , Nina Purpleton & 9!!!";
        char *html = 0;
        size_t html_len = 0;
        char *scannable = 0;
        size_t scannable_len = 0;
        size_t *position_map = 0;
        size_t position_map_len = 0;
        size_t out_idx = 0;

        if (to_html(raw, strlen(raw), 0, &html, &html_len, &out_idx) < 0) {
                goto done;
        }

        if (strcmp(html, correct_html)) {
                ERROR_MESSAGE("Was expecting html conversion to yield "
                              "\n\n\u00ab%s\u00bb\n\nInstead, got "
                              "\n\n\u00ab%s\u00bb\n\n",
                              correct_html, html);
                goto done;
        }

        if (to_scannable(raw, strlen(raw), &scannable, &scannable_len,
                         &position_map, &position_map_len) < 0) {
                goto done;
        }

        if (strcmp(scannable, correct_scannable)) {
                ERROR_MESSAGE("Was expecting scannable conversion to yield "
                              "\n\n\u00ab%s\u00bb\n\nInstead, got "
                              "\n\n\u00ab%s\u00bb\n\n",
                              correct_scannable, scannable);
                goto done;
        }

        if (!(wordfilters = calloc(conf->wordfilter_inputs_num,
                                   sizeof *wordfilters))) {
                PERROR_MESSAGE("calloc");
                goto done;
        }

        wordfilters_num = conf->wordfilter_inputs_num;
        int err_code = 0;
        PCRE2_SIZE err_offset = 0;
        PCRE2_UCHAR8 err_buf[120];

        for (size_t j = 0; j < wordfilters_num; ++j) {
                wordfilters[j].replacement =
                        conf->wordfilter_inputs[j].replacement;
                wordfilters[j].replacement_len = strlen(
                        conf->wordfilter_inputs[j].replacement);

                if ((wordfilters[j].code = pcre2_compile(
                             (PCRE2_SPTR8) conf->wordfilter_inputs[j].pattern,
                             PCRE2_ZERO_TERMINATED, PCRE2_UTF, &err_code,
                             &err_offset, 0))) {
                        continue;
                }

                pcre2_get_error_message(err_code, err_buf, 120);
                ERROR_MESSAGE("pcre2_compile: error with pattern \"%s\": %s",
                              conf->wordfilter_inputs[j].pattern, err_buf);
                goto done;
        }

        const char *format_match_str =

                /* */
                "(?<newline>\\n)"                              /* */
                "|(?<intra_postlink>&gt;&gt;(?<a_num>[0-9]+))" /* */
                "|(?<inter_postlink>&gt;&gt;&gt;/"             /* */
                "(?<e_board>[^ /]+)/(?<e_num>[0-9]+))"         /* */
                "|(?<quote>(?<![^\n])&gt;[^\n]*)";             /* */

        if (!(format_replacements = pcre2_compile(
                      (PCRE2_SPTR8) format_match_str, PCRE2_ZERO_TERMINATED,
                      PCRE2_UTF,
                      &err_code, &err_offset, 0))) {
                pcre2_get_error_message(err_code, err_buf, 120);
                ERROR_MESSAGE("pcre2_compile: error with pattern \"%s\": %s",
                              format_match_str, err_buf);
                goto done;
        }

        ret = 0;
done:
        free(html);
        free(scannable);
        free(position_map);

        return ret;
}

/*
 * Clean up any memory from this file
 *
 * Postconditions (success):
 *
 *  - Valgrind won't report any memory leaks from this file.
 *
 *  - setup_sanitize_comment() can be safely called again.
 */
int clean_sanitize_comment(void)
{
        for (size_t j = 0; j < wordfilters_num; ++j) {
                pcre2_code_free(wordfilters[j].code);
                wordfilters[j] = (struct wordfilter) { 0 };
        }

        pcre2_code_free(format_replacements);
        format_replacements = 0;
        free(wordfilters);
        wordfilters = 0;
        wordfilters_num = 0;

        return 0;
}