diff --git a/Makefile.am b/Makefile.am index e5ea4eb..4171c32 100644 --- a/Makefile.am +++ b/Makefile.am @@ -45,6 +45,7 @@ libisofs_libisofs_la_SOURCES = \ libisofs/filters/xor_encrypt.c \ libisofs/filters/external.c \ libisofs/filters/zisofs.c \ + libisofs/filters/gzip.c \ libisofs/util.h \ libisofs/util.c \ libisofs/util_rbtree.c \ diff --git a/libisofs/filter.h b/libisofs/filter.h index 8143740..a08d266 100644 --- a/libisofs/filter.h +++ b/libisofs/filter.h @@ -24,6 +24,9 @@ /* libisofs/filters/zisofs.c */ #define ISO_FILTER_ZISOFS_DEV_ID 3 +/* libisofs/filters/gzip.c */ +#define ISO_FILTER_GZIP_DEV_ID 4 + typedef struct filter_context FilterContext; diff --git a/libisofs/filters/gzip.c b/libisofs/filters/gzip.c new file mode 100644 index 0000000..1899d72 --- /dev/null +++ b/libisofs/filters/gzip.c @@ -0,0 +1,729 @@ +/* + * Copyright (c) 2009 Thomas Schmitt + * + * This file is part of the libisofs project; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. See COPYING file for details. + * + * It implements a filter facility which can pipe a IsoStream into gzip + * compression resp. uncompression, read its output and forward it as IsoStream + * output to an IsoFile. + * The gzip compression is done via zlib by Jean-loup Gailly and Mark Adler + * who state in : + * "The data format used by the zlib library is described by RFCs (Request for + * Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt + * (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format)." + * + */ + +#include "../libisofs.h" +#include "../filter.h" +#include "../fsource.h" +#include "../util.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef Libisofs_with_zliB +#include +#else +/* If zlib is not available then this code is a dummy */ +#endif + + +/* + * A filter that encodes or decodes the content of gzip compressed files. + */ + + +/* --------------------------- GzipFilterRuntime ------------------------- */ + + +/* Individual runtime properties exist only as long as the stream is opened. + */ +typedef struct +{ + +#ifdef Libisofs_with_zliB + + z_stream strm; /* The zlib processing context */ + +#endif + + char *in_buffer; + char *out_buffer; + int in_buffer_size; + int out_buffer_size; + char *rpt; /* out_buffer + read_bytes */ + + off_t in_counter; + off_t out_counter; + + int do_flush; /* flush mode for deflate() changes at end of input */ + + int error_ret; + +} GzipFilterRuntime; + + +static +int gzip_running_destroy(GzipFilterRuntime **running, int flag) +{ + GzipFilterRuntime *o= *running; + if (o == NULL) + return 0; + if (o->in_buffer != NULL) + free(o->in_buffer); + if (o->out_buffer != NULL) + free(o->out_buffer); + free((char *) o); + *running = NULL; + return 1; +} + + +static +int gzip_running_new(GzipFilterRuntime **running, int flag) +{ + GzipFilterRuntime *o; + + *running = o = calloc(sizeof(GzipFilterRuntime), 1); + if (o == NULL) { + return ISO_OUT_OF_MEM; + } +#ifdef Libisofs_with_zliB + memset(&(o->strm), 0, sizeof(o->strm)); +#endif + o->in_buffer = NULL; + o->out_buffer = NULL; + o->in_buffer_size = 0; + o->out_buffer_size = 0; + o->rpt = NULL; + o->in_counter = 0; + o->out_counter = 0; +#ifdef Libisofs_with_zliB + o->do_flush = Z_NO_FLUSH; +#endif + o->error_ret = 1; + + o->in_buffer_size= 2048; + o->out_buffer_size= 2048; + o->in_buffer = calloc(o->in_buffer_size, 1); + o->out_buffer = calloc(o->out_buffer_size, 1); + if (o->in_buffer == NULL || o->out_buffer == NULL) + goto failed; + o->rpt = o->out_buffer; + return 1; +failed: + gzip_running_destroy(running, 0); + return -1; +} + + +/* ---------------------------- GzipFilterStreamData --------------------- */ + + +/* Counts the number of active compression filters */ +static off_t gzip_ref_count = 0; + +/* Counts the number of active uncompression filters */ +static off_t gunzip_ref_count = 0; + + +#ifdef Libisofs_with_zliB +/* Parameter for deflateInit2() , see */ + +/* >>> ??? get this from zisofs.c ziso_compression_level ? */ +static int gzip_compression_level = 6; + +#endif /* Libisofs_with_zliB */ + + +/* + * The data payload of an individual Gzip Filter IsoStream + */ +typedef struct +{ + IsoStream *orig; + + off_t size; /* -1 means that the size is unknown yet */ + + GzipFilterRuntime *running; /* is non-NULL when open */ + + ino_t id; + +} GzipFilterStreamData; + + + +/* Each individual GzipFilterStreamData needs a unique id number. */ +/* >>> This is very suboptimal: + The counter can rollover. +*/ +static ino_t gzip_ino_id = 0; + +static +int gzip_stream_uncompress(IsoStream *stream, void *buf, size_t desired); + + +/* + * Methods for the IsoStreamIface of a Gzip Filter object. + */ + +/* + * @param flag bit0= original stream is not open + */ +static +int gzip_stream_close_flag(IsoStream *stream, int flag) +{ + +#ifdef Libisofs_with_zliB + + GzipFilterStreamData *data; + + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = stream->data; + + if (data->running == NULL) { + return 1; + } + if (stream->class->read == &gzip_stream_uncompress) { + inflateEnd(&(data->running->strm)); + } else { + deflateEnd(&(data->running->strm)); + } + gzip_running_destroy(&(data->running), 0); + + if (flag & 1) + return 1; + return iso_stream_close(data->orig); + +#else + + return ISO_ZLIB_NOT_ENABLED; + +#endif + +} + + +static +int gzip_stream_close(IsoStream *stream) +{ + return gzip_stream_close_flag(stream, 0); +} + + +/* + * @param flag bit0= do not run .get_size() if size is < 0 + */ +static +int gzip_stream_open_flag(IsoStream *stream, int flag) +{ + +#ifdef Libisofs_with_zliB + + GzipFilterStreamData *data; + GzipFilterRuntime *running = NULL; + int ret; + z_stream *strm; + + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = (GzipFilterStreamData*) stream->data; + if (data->running != NULL) { + return ISO_FILE_ALREADY_OPENED; + } + if (data->size < 0 && !(flag & 1)) { + /* Do the size determination run now, so that the size gets cached + and .get_size() will not fail on an opened stream. + */ + stream->class->get_size(stream); + } + + ret = gzip_running_new(&running, + stream->class->read == &gzip_stream_uncompress); + if (ret < 0) { + return ret; + } + data->running = running; + + /* Start up zlib compression context */ + strm = &(running->strm); + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + if (stream->class->read == &gzip_stream_uncompress) { + ret = inflateInit2(strm, 15 | 16); + } else { + ret = deflateInit2(strm, gzip_compression_level, Z_DEFLATED, + 15 | 16, 8, Z_DEFAULT_STRATEGY); + } + if (ret != Z_OK) + return ISO_ZLIB_COMPR_ERR; + strm->next_out = (Bytef *) running->out_buffer; + strm->avail_out = running->out_buffer_size; + + /* Open input stream */ + ret = iso_stream_open(data->orig); + if (ret < 0) { + return ret; + } + + return 1; + +#else + + return ISO_ZLIB_NOT_ENABLED; + +#endif + +} + + +static +int gzip_stream_open(IsoStream *stream) +{ + return gzip_stream_open_flag(stream, 0); +} + + +/* + * @param flag bit1= uncompress rather than compress + */ +static +int gzip_stream_convert(IsoStream *stream, void *buf, size_t desired, int flag) +{ + +#ifdef Libisofs_with_zliB + + int ret, todo, cnv_ret, c_bytes; + GzipFilterStreamData *data; + GzipFilterRuntime *rng; + size_t fill = 0; + z_stream *strm; + + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = stream->data; + rng= data->running; + if (rng == NULL) { + return ISO_FILE_NOT_OPENED; + } + strm = &(rng->strm); + if (rng->error_ret < 0) { + return rng->error_ret; + } else if (rng->error_ret == 0) { + if (rng->out_buffer_size - strm->avail_out + - (rng->rpt - rng->out_buffer) <= 0) + return 0; + } + + while (1) { + + /* Transfer eventual converted bytes from strm to buf */ + c_bytes = rng->out_buffer_size - strm->avail_out + - (rng->rpt - rng->out_buffer); + if (c_bytes > 0) { + todo = desired - fill; + if (todo > c_bytes) + todo = c_bytes; + memcpy(((char *) buf) + fill, rng->rpt, todo); + rng->rpt += todo; + fill += todo; + rng->out_counter += todo; + } + + if (fill >= desired || rng->error_ret == 0) + return fill; + + /* All buffered out data are consumed now */ + rng->rpt = rng->out_buffer; + strm->next_out = (Bytef *) rng->out_buffer; + strm->avail_out = rng->out_buffer_size; + + if (strm->avail_in == 0) { + /* All pending input is consumed. Get new input. */ + ret = iso_stream_read(data->orig, rng->in_buffer, + rng->in_buffer_size); + if (ret < 0) + return (rng->error_ret = ret); + if (ret == 0) { + if (flag & 2) { + + /* >>> ??? what to do on (early) input EOF ? */; + return (rng->error_ret = ISO_ZLIB_COMPR_ERR); + + } else { + /* Tell zlib by the next call that it is over */ + rng->do_flush = Z_FINISH; + } + } + strm->next_in = (Bytef *) rng->in_buffer; + strm->avail_in = ret; + rng->in_counter += ret; + } + + /* Submit input and fetch output until input is consumed */ + while (1) { + if (flag & 2) { + cnv_ret = inflate(strm, rng->do_flush); + } else { + cnv_ret = deflate(strm, rng->do_flush); + } + if (cnv_ret == Z_STREAM_ERROR || cnv_ret == Z_BUF_ERROR) { + return (rng->error_ret = ISO_ZLIB_COMPR_ERR); + } + if (strm->avail_out < rng->out_buffer_size) + break; /* output is available */ + if (strm->avail_in == 0) /* all pending input consumed */ + break; + } + if (cnv_ret == Z_STREAM_END) + rng->error_ret = 0; + } + return fill; + +#else + + return ISO_ZLIB_NOT_ENABLED; + +#endif + +} + +static +int gzip_stream_compress(IsoStream *stream, void *buf, size_t desired) +{ + return gzip_stream_convert(stream, buf, desired, 0); +} + +static +int gzip_stream_uncompress(IsoStream *stream, void *buf, size_t desired) +{ + return gzip_stream_convert(stream, buf, desired, 2); +} + + +static +off_t gzip_stream_get_size(IsoStream *stream) +{ + int ret, ret_close; + off_t count = 0; + GzipFilterStreamData *data; + char buf[64 * 1024]; + size_t bufsize = 64 * 1024; + + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = stream->data; + + if (data->size >= 0) { + return data->size; + } + + /* Run filter command and count output bytes */ + ret = gzip_stream_open_flag(stream, 1); + if (ret < 0) { + return ret; + } + while (1) { + ret = stream->class->read(stream, buf, bufsize); + if (ret <= 0) + break; + count += ret; + } + ret_close = gzip_stream_close(stream); + if (ret < 0) + return ret; + if (ret_close < 0) + return ret_close; + + data->size = count; + return count; +} + + +static +int gzip_stream_is_repeatable(IsoStream *stream) +{ + /* Only repeatable streams are accepted as orig */ + return 1; +} + + +static +void gzip_stream_get_id(IsoStream *stream, unsigned int *fs_id, + dev_t *dev_id, ino_t *ino_id) +{ + GzipFilterStreamData *data; + + data = stream->data; + *fs_id = ISO_FILTER_FS_ID; + *dev_id = ISO_FILTER_GZIP_DEV_ID; + *ino_id = data->id; +} + + +static +void gzip_stream_free(IsoStream *stream) +{ + GzipFilterStreamData *data; + + if (stream == NULL) { + return; + } + data = stream->data; + if (data->running != NULL) { + gzip_stream_close(stream); + } + if (stream->class->read == &gzip_stream_uncompress) { + if (--gunzip_ref_count < 0) + gunzip_ref_count = 0; + } else { + if (--gzip_ref_count < 0) + gzip_ref_count = 0; + } + iso_stream_unref(data->orig); + free(data); +} + + +static +int gzip_update_size(IsoStream *stream) +{ + /* By principle size is determined only once */ + return 1; +} + + +static +IsoStream *gzip_get_input_stream(IsoStream *stream, int flag) +{ + GzipFilterStreamData *data; + + if (stream == NULL) { + return NULL; + } + data = stream->data; + return data->orig; +} + + +IsoStreamIface gzip_stream_compress_class = { + 2, + "gzip", + gzip_stream_open, + gzip_stream_close, + gzip_stream_get_size, + gzip_stream_compress, + gzip_stream_is_repeatable, + gzip_stream_get_id, + gzip_stream_free, + gzip_update_size, + gzip_get_input_stream +}; + + +IsoStreamIface gzip_stream_uncompress_class = { + 2, + "pizg", + gzip_stream_open, + gzip_stream_close, + gzip_stream_get_size, + gzip_stream_uncompress, + gzip_stream_is_repeatable, + gzip_stream_get_id, + gzip_stream_free, + gzip_update_size, + gzip_get_input_stream +}; + + +static +void gzip_filter_free(FilterContext *filter) +{ + /* no data are allocated */; +} + + +/* + * @param flag bit1= Install a decompression filter + */ +static +int gzip_filter_get_filter(FilterContext *filter, IsoStream *original, + IsoStream **filtered, int flag) +{ + IsoStream *str; + GzipFilterStreamData *data; + + if (filter == NULL || original == NULL || filtered == NULL) { + return ISO_NULL_POINTER; + } + + str = calloc(sizeof(IsoStream), 1); + if (str == NULL) { + return ISO_OUT_OF_MEM; + } + data = calloc(sizeof(GzipFilterStreamData), 1); + if (data == NULL) { + free(str); + return ISO_OUT_OF_MEM; + } + + /* These data items are not owned by this filter object */ + data->id = ++gzip_ino_id; + data->orig = original; + data->size = -1; + data->running = NULL; + + /* get reference to the source */ + iso_stream_ref(data->orig); + + str->refcount = 1; + str->data = data; + if (flag & 2) { + str->class = &gzip_stream_uncompress_class; + gunzip_ref_count++; + } else { + str->class = &gzip_stream_compress_class; + gzip_ref_count++; + } + + *filtered = str; + + return ISO_SUCCESS; +} + + +/* To be called by iso_file_add_filter(). + * The FilterContext input parameter is not furtherly needed for the + * emerging IsoStream. + */ +static +int gzip_filter_get_compressor(FilterContext *filter, IsoStream *original, + IsoStream **filtered) +{ + return gzip_filter_get_filter(filter, original, filtered, 0); +} + +static +int gzip_filter_get_uncompressor(FilterContext *filter, IsoStream *original, + IsoStream **filtered) +{ + return gzip_filter_get_filter(filter, original, filtered, 2); +} + + +/* Produce a parameter object suitable for iso_file_add_filter(). + * It may be disposed by free() after all those calls are made. + * + * This is quite a dummy as it does not carry individual data. + * @param flag bit1= Install a decompression filter + */ +static +int gzip_create_context(FilterContext **filter, int flag) +{ + FilterContext *f; + + *filter = f = calloc(1, sizeof(FilterContext)); + if (f == NULL) { + return ISO_OUT_OF_MEM; + } + f->refcount = 1; + f->version = 0; + f->data = NULL; + f->free = gzip_filter_free; + if (flag & 2) + f->get_filter = gzip_filter_get_uncompressor; + else + f->get_filter = gzip_filter_get_compressor; + return ISO_SUCCESS; +} + + +/* + * @param flag bit0= if_block_reduction rather than if_reduction + * bit1= Install a decompression filter + * bit2= only inquire availability of gzip filtering + * bit3= do not inquire size + */ +int gzip_add_filter(IsoFile *file, int flag) +{ + +#ifdef Libisofs_with_zliB + + int ret; + FilterContext *f = NULL; + IsoStream *stream; + off_t original_size = 0, filtered_size = 0; + + if (flag & 4) + return 2; + + original_size = iso_file_get_size(file); + + ret = gzip_create_context(&f, flag & 2); + if (ret < 0) { + return ret; + } + ret = iso_file_add_filter(file, f, 0); + free(f); + if (ret < 0) { + return ret; + } + if (flag & 8) /* size will be filled in by caller */ + return ISO_SUCCESS; + + /* Run a full filter process getsize so that the size is cached */ + stream = iso_file_get_stream(file); + filtered_size = iso_stream_get_size(stream); + if (filtered_size < 0) { + iso_file_remove_filter(file, 0); + return filtered_size; + } + if ((filtered_size >= original_size || + ((flag & 1) && filtered_size / 2048 >= original_size / 2048)) + && !(flag & 2)){ + ret = iso_file_remove_filter(file, 0); + if (ret < 0) { + return ret; + } + return 2; + } + return ISO_SUCCESS; + +#else + + return ISO_ZLIB_NOT_ENABLED; + +#endif /* ! Libisofs_with_zliB */ + +} + + +/* API function */ +int iso_file_add_gzip_filter(IsoFile *file, int flag) +{ + return gzip_add_filter(file, flag & ~8); +} + + +/* API function */ +int iso_gzip_get_refcounts(off_t *gzip_count, off_t *gunzip_count, int flag) +{ + *gzip_count = gzip_ref_count; + *gunzip_count = gunzip_ref_count; + return ISO_SUCCESS; +} + diff --git a/libisofs/filters/zisofs.c b/libisofs/filters/zisofs.c index 62a0abc..7494d03 100644 --- a/libisofs/filters/zisofs.c +++ b/libisofs/filters/zisofs.c @@ -212,7 +212,7 @@ static ino_t ziso_ino_id = 0; /* - * Methods for the IsoStreamIface of an External Filter object. + * Methods for the IsoStreamIface of an Zisofs Filter object. */ static diff --git a/libisofs/libisofs.h b/libisofs/libisofs.h index a218856..43c5bb8 100644 --- a/libisofs/libisofs.h +++ b/libisofs/libisofs.h @@ -4749,6 +4749,7 @@ int iso_local_set_attrs(char *disk_path, size_t num_attrs, char **names, * iso_file_add_external_filter() * and internal filters * iso_file_add_zisofs_filter() + * iso_file_add_gzip_filter() * which may or may not be available depending on compile time settings and * installed software packages like libz. * @@ -4913,7 +4914,7 @@ int iso_stream_get_external_filter(IsoStream *stream, /* ts A90409 */ /** - * Install a zisofs filter on top of the content stream of a data * file. + * Install a zisofs filter on top of the content stream of a data file. * zisofs is a compression format which is decompressed by some Linux kernels. * See also doc/zisofs_format.txt . * The filter will not be installed if its output size is not smaller than @@ -5040,6 +5041,50 @@ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag); int iso_node_zf_by_magic(IsoNode *node, int flag); +/* ts A90414 */ +/** + * Install a gzip or gunzip filter on top of the content stream of a data file. + * gzip is a compression format which is used by programs gzip and gunzip. + * The filter will not be installed if its output size is not smaller than + * the size of the input stream. + * This is only enabled if the use of libz was enabled at compile time. + * @param file + * The data file node which shall show filtered content. + * @param flag + * Bitfield for control purposes + * bit0= Do not install filter if the number of output blocks is + * not smaller than the number of input blocks. Block size is 2048. + * bit1= Install a decompression filter rather than one for compression. + * bit2= Only inquire availability of gzip filtering. file may be NULL. + * If available return 2, else return error. + * bit3= is reserved for internal use and will be forced to 0 + * @return + * 1 on success, 2 if filter available but installation revoked + * <0 on error, e.g. ISO_ZLIB_NOT_ENABLED + * + * @since 0.6.18 + */ +int iso_file_add_gzip_filter(IsoFile *file, int flag); + + +/* ts A90414 */ +/** + * Inquire the number of gzip compression and uncompression filters which + * are in use. + * @param gzip_count + * Will return the number of currently installed compression filters. + * @param gunzip_count + * Will return the number of currently installed uncompression filters. + * @param flag + * Bitfield for control purposes, unused yet, submit 0 + * @return + * 1 on success, <0 on error + * + * @since 0.6.18 + */ +int iso_gzip_get_refcounts(off_t *gzip_count, off_t *gunzip_count, int flag); + + /* ------------------------------------------------------------------------- */ #ifdef LIBISOFS_WITHOUT_LIBBURN