From 6eba447e8ae12cacac3b479c7148d189bea3e8ea Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Mon, 12 Oct 2015 23:18:23 +0200 Subject: [PATCH] Experimental brotli encoding support. https://github.com/bagder/libbrotli --- Makefile.config.in | 1 + configure.in | 5 +- src/encoding/Makefile | 2 + src/encoding/brotli.c | 203 +++++++++++++++++++++++++++++++++++++++++++++++ src/encoding/brotli.h | 12 +++ src/encoding/encoding.c | 2 + src/encoding/encoding.h | 1 + src/protocol/http/http.c | 16 +++- 8 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 src/encoding/brotli.c create mode 100644 src/encoding/brotli.h diff --git a/Makefile.config.in b/Makefile.config.in index ada7e074..cb3ce7c5 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -109,6 +109,7 @@ CONFIG_DOXYGEN = @CONFIG_DOXYGEN@ CONFIG_BACKTRACE = @CONFIG_BACKTRACE@ CONFIG_BITTORRENT = @CONFIG_BITTORRENT@ CONFIG_BOOKMARKS = @CONFIG_BOOKMARKS@ +CONFIG_BROTLI = @CONFIG_BROTLI@ CONFIG_BZIP2 = @CONFIG_BZIP2@ CONFIG_CGI = @CONFIG_CGI@ CONFIG_COOKIES = @CONFIG_COOKIES@ diff --git a/configure.in b/configure.in index 58548fef..ded9c9e3 100644 --- a/configure.in +++ b/configure.in @@ -483,6 +483,9 @@ EL_CONFIG_OPTIONAL_LIBRARY(CONFIG_GZIP, zlib, zlib.h, z, gzclearerr, EL_CONFIG_OPTIONAL_LIBRARY(CONFIG_BZIP2, bzlib, bzlib.h, bz2, BZ2_bzReadOpen, [ --without-bzlib disable bzlib support]) +EL_CONFIG_OPTIONAL_LIBRARY(CONFIG_BROTLI, brotli, brotli/dec/decode.h, brotlidec, BrotliStateInit, + [ --with-brotli enable experimental brotli support]) + EL_CONFIG_OPTIONAL_LIBRARY(CONFIG_IDN, idn, idna.h, idn, stringprep_check_version, [ --without-idn disable international domain names support]) @@ -1694,7 +1697,7 @@ if test "x$ac_cv_c_compiler_gnu" = "xyes"; then 4.5*) CFLAGS="$CFLAGS -fno-strict-aliasing -Wno-pointer-sign -Wno-enum-compare" ;; - 4.*) + 4.*|5.*) # Do not show warnings related to (char * | unsigned char *) type # difference. CFLAGS="$CFLAGS -fno-strict-aliasing -Wno-pointer-sign" diff --git a/src/encoding/Makefile b/src/encoding/Makefile index 61ed5e64..0615bae9 100644 --- a/src/encoding/Makefile +++ b/src/encoding/Makefile @@ -1,6 +1,8 @@ top_builddir=../.. include $(top_builddir)/Makefile.config + +OBJS-$(CONFIG_BROTLI) += brotli.o OBJS-$(CONFIG_BZIP2) += bzip2.o OBJS-$(CONFIG_GZIP) += deflate.o OBJS-$(CONFIG_LZMA) += lzma.o diff --git a/src/encoding/brotli.c b/src/encoding/brotli.c new file mode 100644 index 00000000..9dc1fe69 --- /dev/null +++ b/src/encoding/brotli.c @@ -0,0 +1,203 @@ +/* Brotli encoding (ENCODING_BROTLI) backend */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_BROTLI_DEC_DECODE_H +#include +#endif + +#include + +#include "elinks.h" + +#include "encoding/brotli.h" +#include "encoding/encoding.h" +#include "util/math.h" +#include "util/memory.h" + +struct br_enc_data { + BrotliState br_stream; + + uint8_t *input; + uint8_t *output; + + size_t input_length; + size_t output_length; + size_t output_pos; + size_t input_pos; + + /* The file descriptor from which we read. */ + int fdread; + int after_end:1; + int need_free:1; +}; + +static int +brotli_open(struct stream_encoded *stream, int fd) +{ + struct br_enc_data *data = mem_calloc(1, sizeof(*data)); + + stream->data = NULL; + if (!data) { + return -1; + } + + data->fdread = fd; + BrotliStateInit(&data->br_stream); + stream->data = data; + + return 0; +} + +static int +brotli_read_function_fd(void *data, uint8_t *buf, size_t len) +{ + struct br_enc_data *enc_data = (struct br_enc_data *)data; + + return safe_read(enc_data->fdread, buf, len); +} + +static int +brotli_read_function(void *data, uint8_t *buf, size_t len) +{ + struct br_enc_data *enc_data = (struct br_enc_data *)data; + size_t l = MIN(len, enc_data->input_length - enc_data->input_pos); + + memcpy(buf, enc_data->input + enc_data->input_pos, l); + enc_data->input_pos += l; + return l; +} + +static int +brotli_write_function(void *data, const uint8_t *buf, size_t len) +{ + struct br_enc_data *enc_data = (struct br_enc_data *)data; + + enc_data->output = mem_alloc(len); + if (!enc_data->output) { + return -1; + } + memcpy(enc_data->output, buf, len); + enc_data->output_length = len; + return len; +} + +static int +brotli_read(struct stream_encoded *stream, unsigned char *buf, int len) +{ + struct br_enc_data *enc_data = (struct br_enc_data *) stream->data; + BrotliState *s; + BrotliInput inp; + BrotliOutput outp; + size_t l; + int error; + + if (!enc_data) return -1; + s = &enc_data->br_stream; + + assert(len > 0); + + if (enc_data->after_end) { + l = MIN(len, enc_data->output_length - enc_data->output_pos); + memcpy(buf, enc_data->output + enc_data->output_pos, l); + enc_data->output_pos += l; + return l; + } + + enc_data->input = NULL; + enc_data->input_length = 0; + enc_data->output = NULL; + enc_data->output_length = 0; + enc_data->output_pos = 0; + inp.data_ = enc_data; + outp.data_ = enc_data; + inp.cb_ = brotli_read_function_fd; + outp.cb_ = brotli_write_function; + + error = BrotliDecompressStreaming(inp, outp, 1, s); + switch (error) { + case BROTLI_RESULT_ERROR: + return -1; + case BROTLI_RESULT_SUCCESS: + enc_data->after_end = 1; + case BROTLI_RESULT_NEEDS_MORE_INPUT: + default: + enc_data->need_free = 1; + l = MIN(len, enc_data->output_length - enc_data->output_pos); + memcpy(buf, enc_data->output + enc_data->output_pos, l); + enc_data->output_pos += l; + return l; + } +} + +static unsigned char * +brotli_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len) +{ + struct br_enc_data *enc_data = (struct br_enc_data *)st->data; + BrotliInput inp; + BrotliOutput outp; + BrotliState *stream = &enc_data->br_stream; + int error; + int finish = (len == 0); + + *new_len = 0; /* default, left there if an error occurs */ + enc_data->input = data; + enc_data->input_length = len; + enc_data->input_pos = 0; + enc_data->output = NULL; + enc_data->output_length = 0; + enc_data->output_pos = 0; + inp.data_ = enc_data; + outp.data_ = enc_data; + inp.cb_ = brotli_read_function; + outp.cb_ = brotli_write_function; + error = BrotliDecompressStreaming(inp, outp, finish, stream); + + switch (error) { + case BROTLI_RESULT_ERROR: + return NULL; + case BROTLI_RESULT_SUCCESS: + enc_data->after_end = 1; + case BROTLI_RESULT_NEEDS_MORE_INPUT: + default: + *new_len = enc_data->output_length; + return enc_data->output; + } +} + +static void +brotli_close(struct stream_encoded *stream) +{ + struct br_enc_data *data = (struct br_enc_data *) stream->data; + + if (data) { + BrotliStateCleanup(&data->br_stream); + if (data->fdread != -1) { + close(data->fdread); + } + if (data->need_free) { + mem_free_if(data->output); + } + mem_free(data); + stream->data = 0; + } +} + +static const unsigned char *const brotli_extensions[] = { ".br", NULL }; + +const struct decoding_backend brotli_decoding_backend = { + "brotli", + brotli_extensions, + brotli_open, + brotli_read, + brotli_decode_buffer, + brotli_close, +}; diff --git a/src/encoding/brotli.h b/src/encoding/brotli.h new file mode 100644 index 00000000..4f2b4247 --- /dev/null +++ b/src/encoding/brotli.h @@ -0,0 +1,12 @@ +#ifndef EL__ENCODING_BROTLI_H +#define EL__ENCODING_BROTLI_H + +#include "encoding/encoding.h" + +#ifdef CONFIG_BROTLI +extern const struct decoding_backend brotli_decoding_backend; +#else +#define brotli_decoding_backend dummy_decoding_backend +#endif + +#endif diff --git a/src/encoding/encoding.c b/src/encoding/encoding.c index 440df019..d072da7f 100644 --- a/src/encoding/encoding.c +++ b/src/encoding/encoding.c @@ -83,6 +83,7 @@ static const struct decoding_backend dummy_decoding_backend = { /* Dynamic backend area */ +#include "encoding/brotli.h" #include "encoding/bzip2.h" #include "encoding/deflate.h" #include "encoding/lzma.h" @@ -93,6 +94,7 @@ static const struct decoding_backend *const decoding_backends[] = { &bzip2_decoding_backend, &lzma_decoding_backend, &deflate_decoding_backend, + &brotli_decoding_backend, }; diff --git a/src/encoding/encoding.h b/src/encoding/encoding.h index 89246371..f196c2c0 100644 --- a/src/encoding/encoding.h +++ b/src/encoding/encoding.h @@ -10,6 +10,7 @@ enum stream_encoding { ENCODING_BZIP2, ENCODING_LZMA, ENCODING_DEFLATE, + ENCODING_BROTLI, /* Max. number of known encoding including ENCODING_NONE. */ ENCODINGS_KNOWN, diff --git a/src/protocol/http/http.c b/src/protocol/http/http.c index 611a94e9..69c5347b 100644 --- a/src/protocol/http/http.c +++ b/src/protocol/http/http.c @@ -588,12 +588,18 @@ init_http_connection_info(struct connection *conn, int major, int minor, int clo static void accept_encoding_header(struct string *header) { -#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) || defined(CONFIG_LZMA) +#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) || defined(CONFIG_LZMA) || defined(CONFIG_BROTLI) int comma = 0; add_to_string(header, "Accept-Encoding: "); +#ifdef CONFIG_BROTLI + add_to_string(header, "br"); + comma = 1; +#endif + #ifdef CONFIG_BZIP2 + if (comma) add_to_string(header, ", "); add_to_string(header, "bzip2"); comma = 1; #endif @@ -1850,7 +1856,7 @@ again: d = parse_header(conn->cached->head, "Content-Encoding", NULL); if (d) { -#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) || defined(CONFIG_LZMA) +#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) || defined(CONFIG_LZMA) || defined(CONFIG_BROTLI) unsigned char *extension = get_extension_from_uri(uri); enum stream_encoding file_encoding; @@ -1868,6 +1874,12 @@ again: conn->content_encoding = ENCODING_DEFLATE; #endif +#ifdef CONFIG_BROTLI + if (file_encoding != ENCODING_BROTLI + && (!c_strcasecmp(d, "br"))) + conn->content_encoding = ENCODING_BROTLI; +#endif + #ifdef CONFIG_BZIP2 if (file_encoding != ENCODING_BZIP2 && (!c_strcasecmp(d, "bzip2") || !c_strcasecmp(d, "x-bzip2"))) -- 2.11.4.GIT