From 24995eb239799e52ae09d47b7520d50a3b8b606c Mon Sep 17 00:00:00 2001 From: Allen Wild Date: Tue, 5 Sep 2017 19:48:49 -0400 Subject: [PATCH] zfile: fix build when zlib and/or lzma are excluded Currently, zfile won't build if either zlib.h or lzma.h are missing * Add several inline wrapper functions which check cookie->ctype and call an inline zlib or lzma implementation function. - This cleans up zfile_read at the expense of growing the file and adding 2 layers of function calls. Everything is static and inline so the compiler should be able to optimize sufficiently. * Stub functions are #define'd for the unsupported compression method (if applicable) * Use an Automake conditional to only build zfile.c if either zlib.h or lzma.h are available, define the symbol USE_FOPENCOOKIE if this is the case and check it instead of HAVE_FOPENCOOKIE * Replace tabs in zfile.c with spaces for consistency with the rest of the codebase Fixes: #1147 --- Makefile.am | 6 +- configure.ac | 5 + src/decompress.h | 2 +- src/search.c | 2 +- src/zfile.c | 301 ++++++++++++++++++++++++++++++++++++++++--------------- 5 files changed, 232 insertions(+), 84 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3931c3a7..4d85f54f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,9 +1,13 @@ ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS} bin_PROGRAMS = ag -ag_SOURCES = src/ignore.c src/ignore.h src/log.c src/log.h src/options.c src/options.h src/print.c src/print_w32.c src/print.h src/scandir.c src/scandir.h src/search.c src/search.h src/lang.c src/lang.h src/util.c src/util.h src/decompress.c src/decompress.h src/uthash.h src/main.c src/zfile.c +ag_SOURCES = src/ignore.c src/ignore.h src/log.c src/log.h src/options.c src/options.h src/print.c src/print_w32.c src/print.h src/scandir.c src/scandir.h src/search.c src/search.h src/lang.c src/lang.h src/util.c src/util.h src/decompress.c src/decompress.h src/uthash.h src/main.c ag_LDADD = ${PCRE_LIBS} ${LZMA_LIBS} ${ZLIB_LIBS} $(PTHREAD_LIBS) +if USE_FOPENCOOKIE +ag_SOURCES += src/zfile.c src/zfile.h +endif + dist_man_MANS = doc/ag.1 bashcompdir = $(pkgdatadir)/completions diff --git a/configure.ac b/configure.ac index 982cc3b2..5917ecc1 100644 --- a/configure.ac +++ b/configure.ac @@ -66,6 +66,11 @@ AC_CHECK_MEMBER([struct dirent.d_namlen], [AC_DEFINE([HAVE_DIRENT_DNAMLEN], [], AC_CHECK_FUNCS(fgetln fopencookie getline realpath strlcpy strndup vasprintf madvise posix_fadvise pthread_setaffinity_np pledge) +# Only build zfile.c if we need it +AM_CONDITIONAL([USE_FOPENCOOKIE], [(test "$ac_cv_func_fopencookie" = "yes") && + (test "$ac_cv_header_zlib_h" = "yes" || test "$ac_cv_header_lzma_h" = "yes")]) +AM_COND_IF([USE_FOPENCOOKIE], [AC_DEFINE([USE_FOPENCOOKIE], [], [Use fopencookie streaming in zfile.c])]) + AC_CONFIG_FILES([Makefile the_silver_searcher.spec]) AC_CONFIG_HEADERS([src/config.h]) diff --git a/src/decompress.h b/src/decompress.h index 9c5592cf..d5c3d582 100644 --- a/src/decompress.h +++ b/src/decompress.h @@ -19,7 +19,7 @@ ag_compression_type is_zipped(const void *buf, const int buf_len); void *decompress(const ag_compression_type zip_type, const void *buf, const int buf_len, const char *dir_full_path, int *new_buf_len); -#if HAVE_FOPENCOOKIE +#ifdef USE_FOPENCOOKIE FILE *decompress_open(int fd, const char *mode, ag_compression_type ctype); #endif diff --git a/src/search.c b/src/search.c index 14e9d415..84d9423f 100644 --- a/src/search.c +++ b/src/search.c @@ -357,7 +357,7 @@ void search_file(const char *file_full_path) { if (opts.search_zip_files) { ag_compression_type zip_type = is_zipped(buf, f_len); if (zip_type != AG_NO_COMPRESSION) { -#if HAVE_FOPENCOOKIE +#ifdef USE_FOPENCOOKIE log_debug("%s is a compressed file. stream searching", file_full_path); fp = decompress_open(fd, "r", zip_type); search_stream(fp, file_full_path); diff --git a/src/zfile.c b/src/zfile.c index e4b75662..74de37e6 100644 --- a/src/zfile.c +++ b/src/zfile.c @@ -33,10 +33,10 @@ typedef _off64_t off64_t; #if HAVE_FOPENCOOKIE -#define min(a, b) ({ \ - __typeof (a) _a = (a); \ - __typeof (b) _b = (b); \ - _a < _b ? _a : _b; }) +#define min(a, b) ({ \ + __typeof (a) _a = (a); \ + __typeof (b) _b = (b); \ + _a < _b ? _a : _b; }) static cookie_read_function_t zfile_read; static cookie_seek_function_t zfile_seek; @@ -60,23 +60,217 @@ struct zfile { ag_compression_type ctype; union { +#ifdef HAVE_ZLIB_H z_stream gz; +#endif +#ifdef HAVE_LZMA_H lzma_stream lzma; +#endif } stream; uint8_t inbuf[32 * KB]; uint8_t outbuf[256 * KB]; bool eof; + bool eof_next; }; -#define CAVAIL_IN(c) ((c)->ctype == AG_GZIP ? (c)->stream.gz.avail_in : (c)->stream.lzma.avail_in) -#define CNEXT_OUT(c) ((c)->ctype == AG_GZIP ? (c)->stream.gz.next_out : (c)->stream.lzma.next_out) +#ifdef HAVE_ZLIB_H +/***************** zlib functions ********************************/ +static int zfile_zlib_cookie_init(struct zfile *cookie) { + int rc; + + memset(&cookie->stream.gz, 0, sizeof(cookie->stream.gz)); + rc = inflateInit2(&cookie->stream.gz, 32 + 15); + if (rc != Z_OK) { + log_err("Unable to initialize zlib: %s", zError(rc)); + return EIO; + } + cookie->stream.gz.next_in = NULL; + cookie->stream.gz.avail_in = 0; + cookie->stream.gz.next_out = cookie->outbuf; + cookie->stream.gz.avail_out = sizeof cookie->outbuf; + return 0; +} + +static inline size_t zfile_zlib_cavail_in(struct zfile *cookie) { + return (size_t)cookie->stream.gz.avail_in; +} + +static inline uint8_t *zfile_zlib_cnext_out(struct zfile *cookie) { + return (uint8_t *)cookie->stream.gz.next_out; +} + +static inline int zfile_zlib_is_stream_end(int ret) { + return ret == Z_STREAM_END; +} + +static inline void zfile_zlib_set_next_in(struct zfile *cookie, size_t nb) { + cookie->stream.gz.avail_in = nb; + cookie->stream.gz.next_in = cookie->inbuf; +} + +static inline void zfile_zlib_set_next_out(struct zfile *cookie) { + cookie->stream.gz.next_out = cookie->outbuf; + cookie->stream.gz.avail_out = sizeof(cookie->outbuf); +} + +static inline int zfile_zlib_inflate(struct zfile *cookie) { + int ret = inflate(&cookie->stream.gz, Z_NO_FLUSH); + if (ret == Z_STREAM_END) { + cookie->eof_next = true; + return 0; + } + return ret == Z_OK ? 0 : -1; +} + +#else +/***************** zlib stubs ********************************/ +#define zfile_zlib_cookie_init(c) EINVAL +#define zfile_zlib_cavail_in(c) 0 +#define zfile_zlib_cnext_out(c) 0 +#define zfile_zlib_is_stream_end(r) 1 +#define zfile_zlib_set_next_in(c, n) (void)0 +#define zfile_zlib_set_next_out(c) (void)0 +#define zfile_zlib_inflate(c) -1 +#endif -static int -zfile_cookie_init(struct zfile *cookie) { #ifdef HAVE_LZMA_H +/***************** lzma functions ********************************/ +static int zfile_lzma_cookie_init(struct zfile *cookie) { lzma_ret lzrc; + cookie->stream.lzma = (lzma_stream)LZMA_STREAM_INIT; + lzrc = lzma_auto_decoder(&cookie->stream.lzma, -1, 0); + if (lzrc != LZMA_OK) { + log_err("Unable to initialize lzma_auto_decoder: %d", lzrc); + return EIO; + } + cookie->stream.lzma.next_in = NULL; + cookie->stream.lzma.avail_in = 0; + cookie->stream.lzma.next_out = cookie->outbuf; + cookie->stream.lzma.avail_out = sizeof(cookie->outbuf); + return 0; +} + +static inline size_t zfile_lzma_cavail_in(struct zfile *cookie) { + return cookie->stream.lzma.avail_in; +} + +static inline uint8_t *zfile_lzma_cnext_out(struct zfile *cookie) { + return cookie->stream.lzma.next_out; +} + +static inline int zfile_lzma_is_stream_end(int ret) { + return (lzma_ret)ret == LZMA_STREAM_END; +} + +static inline void zfile_lzma_set_next_in(struct zfile *cookie, size_t nb) { + cookie->stream.lzma.avail_in = nb; + cookie->stream.lzma.next_in = cookie->inbuf; +} + +static inline void zfile_lzma_set_next_out(struct zfile *cookie) { + cookie->stream.lzma.next_out = cookie->outbuf; + cookie->stream.lzma.avail_out = sizeof(cookie->outbuf); +} + +static inline int zfile_lzma_inflate(struct zfile *cookie) { + lzma_ret ret = lzma_code(&cookie->stream.lzma, LZMA_RUN); + if (ret == LZMA_STREAM_END) { + cookie->eof_next = true; + return 0; + } + return ret == LZMA_OK ? 0 : -1; +} + +#else +/***************** lzma stubs ********************************/ +#define zfile_lzma_cookie_init(c) EINVAL +#define zfile_lzma_cavail_in(c) 0 +#define zfile_lzma_cnext_out(c) 0 +#define zfile_lzma_is_stream_end(r) 1 +#define zfile_lzma_set_next_in(c, n) (void)0 +#define zfile_lzma_set_next_out(c) (void)0 +#define zfile_lzma_inflate(c) -1 #endif + +static inline size_t zfile_cavail_in(struct zfile *cookie) { + switch (cookie->ctype) { + case AG_GZIP: + return zfile_zlib_cavail_in(cookie); + break; + case AG_XZ: + return zfile_lzma_cavail_in(cookie); + break; + default: + return 0; + } +} + +static inline uint8_t *zfile_cnext_out(struct zfile *cookie) { + switch (cookie->ctype) { + case AG_GZIP: + return zfile_zlib_cnext_out(cookie); + break; + case AG_XZ: + return zfile_lzma_cnext_out(cookie); + break; + default: + return NULL; + } +} + +static inline int zfile_is_stream_end(struct zfile *cookie, int ret) { + switch (cookie->ctype) { + case AG_GZIP: + return zfile_zlib_is_stream_end(ret); + case AG_XZ: + return zfile_lzma_is_stream_end(ret); + default: + return 1; + } +} + +static inline void zfile_set_next_in(struct zfile *cookie, size_t nb) { + switch (cookie->ctype) { + case AG_GZIP: + zfile_zlib_set_next_in(cookie, nb); + break; + case AG_XZ: + zfile_lzma_set_next_in(cookie, nb); + break; + default: + break; + } +} + +static inline void zfile_set_next_out(struct zfile *cookie) { + switch (cookie->ctype) { + case AG_GZIP: + zfile_zlib_set_next_out(cookie); + break; + case AG_XZ: + zfile_lzma_set_next_out(cookie); + break; + default: + break; + } +} + +static inline int zfile_inflate(struct zfile *cookie) { + switch (cookie->ctype) { + case AG_GZIP: + return zfile_zlib_inflate(cookie); + break; + case AG_XZ: + return zfile_lzma_inflate(cookie); + break; + default: + return -1; + } +} + +static int +zfile_cookie_init(struct zfile *cookie) { int rc; assert(cookie->logic_offset == 0); @@ -85,39 +279,18 @@ zfile_cookie_init(struct zfile *cookie) { cookie->actual_len = 0; switch (cookie->ctype) { -#ifdef HAVE_ZLIB_H case AG_GZIP: - memset(&cookie->stream.gz, 0, sizeof cookie->stream.gz); - rc = inflateInit2(&cookie->stream.gz, 32 + 15); - if (rc != Z_OK) { - log_err("Unable to initialize zlib: %s", zError(rc)); - return EIO; - } - cookie->stream.gz.next_in = NULL; - cookie->stream.gz.avail_in = 0; - cookie->stream.gz.next_out = cookie->outbuf; - cookie->stream.gz.avail_out = sizeof cookie->outbuf; + rc = zfile_zlib_cookie_init(cookie); break; -#endif -#ifdef HAVE_LZMA_H case AG_XZ: - cookie->stream.lzma = (lzma_stream)LZMA_STREAM_INIT; - lzrc = lzma_auto_decoder(&cookie->stream.lzma, -1, 0); - if (lzrc != LZMA_OK) { - log_err("Unable to initialize lzma_auto_decoder: %d", lzrc); - return EIO; - } - cookie->stream.lzma.next_in = NULL; - cookie->stream.lzma.avail_in = 0; - cookie->stream.lzma.next_out = cookie->outbuf; - cookie->stream.lzma.avail_out = sizeof cookie->outbuf; + rc = zfile_lzma_cookie_init(cookie); break; -#endif default: log_err("Unsupported compression type: %d", cookie->ctype); return EINVAL; } - + if (rc) + return rc; cookie->outbuf_start = 0; cookie->eof = false; @@ -165,10 +338,10 @@ decompress_open(int fd, const char *mode, ag_compression_type ctype) { goto out; /* - * No validation of compression type is done -- file is assumed to - * match input. In Ag, the compression type is already detected, so - * that's ok. - */ + * No validation of compression type is done -- file is assumed to + * match input. In Ag, the compression type is already detected, so + * that's ok. + */ cookie = malloc(sizeof *cookie); if (cookie == NULL) { errno = ENOMEM; @@ -207,8 +380,6 @@ zfile_read(void *cookie_, char *buf, size_t size) { struct zfile *cookie = cookie_; size_t nb, ignorebytes; ssize_t total = 0; - lzma_ret lzret; - int ret; assert(size <= SSIZE_MAX); @@ -218,9 +389,6 @@ zfile_read(void *cookie_, char *buf, size_t size) { if (cookie->eof) return 0; - ret = Z_OK; - lzret = LZMA_OK; - ignorebytes = cookie->logic_offset - cookie->decode_offset; assert(ignorebytes == 0); @@ -228,9 +396,9 @@ zfile_read(void *cookie_, char *buf, size_t size) { size_t inflated; /* Drain output buffer first */ - while (CNEXT_OUT(cookie) > + while (zfile_cnext_out(cookie) > &cookie->outbuf[cookie->outbuf_start]) { - size_t left = CNEXT_OUT(cookie) - + size_t left = zfile_cnext_out(cookie) - &cookie->outbuf[cookie->outbuf_start]; size_t ignoreskip = min(ignorebytes, left); size_t toread; @@ -265,21 +433,13 @@ zfile_read(void *cookie_, char *buf, size_t size) { if (size == 0) break; - /* - * If we have not satisfied read, the output buffer must be - * empty. - */ - assert(cookie->stream.gz.next_out == - &cookie->outbuf[cookie->outbuf_start]); - - if ((cookie->ctype == AG_XZ && lzret == LZMA_STREAM_END) || - (cookie->ctype == AG_GZIP && ret == Z_STREAM_END)) { + if (cookie->eof_next) { cookie->eof = true; break; } /* Read more input if empty */ - if (CAVAIL_IN(cookie) == 0) { + if (zfile_cavail_in(cookie) == 0) { nb = fread(cookie->inbuf, 1, sizeof cookie->inbuf, cookie->in); if (ferror(cookie->in)) { @@ -290,39 +450,18 @@ zfile_read(void *cookie_, char *buf, size_t size) { warn("truncated file"); exit(1); } - if (cookie->ctype == AG_XZ) { - cookie->stream.lzma.avail_in = nb; - cookie->stream.lzma.next_in = cookie->inbuf; - } else { - cookie->stream.gz.avail_in = nb; - cookie->stream.gz.next_in = cookie->inbuf; - } + zfile_set_next_in(cookie, nb); } /* Reset stream state to beginning of output buffer */ - if (cookie->ctype == AG_XZ) { - cookie->stream.lzma.next_out = cookie->outbuf; - cookie->stream.lzma.avail_out = sizeof cookie->outbuf; - } else { - cookie->stream.gz.next_out = cookie->outbuf; - cookie->stream.gz.avail_out = sizeof cookie->outbuf; - } + zfile_set_next_out(cookie); cookie->outbuf_start = 0; - if (cookie->ctype == AG_GZIP) { - ret = inflate(&cookie->stream.gz, Z_NO_FLUSH); - if (ret != Z_OK && ret != Z_STREAM_END) { - log_err("Found mem/data error while decompressing zlib stream: %s", zError(ret)); - return -1; - } - } else { - lzret = lzma_code(&cookie->stream.lzma, LZMA_RUN); - if (lzret != LZMA_OK && lzret != LZMA_STREAM_END) { - log_err("Found mem/data error while decompressing xz/lzma stream: %d", lzret); - return -1; - } + if (zfile_inflate(cookie)) { + log_err("Found mem/data error while decompressing stream"); + return -1; } - inflated = CNEXT_OUT(cookie) - &cookie->outbuf[0]; + inflated = zfile_cnext_out(cookie) - &cookie->outbuf[0]; cookie->actual_len += inflated; } while (!ferror(cookie->in) && size > 0);