From 2ca3b292fb0afdbe3c2554f09d8b90dd4f8f64f6 Mon Sep 17 00:00:00 2001 From: Thomas Schmitt Date: Sat, 17 Oct 2020 14:40:24 +0200 Subject: [PATCH] New iso_zisofs_ctrl parameter .block_number_target --- libisofs/filters/zisofs.c | 93 +++++++++++++++++++++++++-------------- libisofs/libisofs.h | 20 ++++++++- libisofs/messages.c | 2 + 3 files changed, 80 insertions(+), 35 deletions(-) diff --git a/libisofs/filters/zisofs.c b/libisofs/filters/zisofs.c index 50b5a73..44e055d 100644 --- a/libisofs/filters/zisofs.c +++ b/libisofs/filters/zisofs.c @@ -84,21 +84,58 @@ static uint64_t ziso_block_pointer_mgt(uint64_t num, int mode); #define ISO_ZISOFS_V2_MAX_LOG2 20 -/* --------------------------- ZisofsFilterRuntime ------------------------- */ - +/* --------------------------- Runtime parameters ------------------------- */ /* Sizes to be used for compression. Decompression learns from input header. */ static uint8_t ziso_block_size_log2 = 15; -static int ziso_block_size = 32768; static int ziso_v2_enabled = 0; static int ziso_v2_block_size_log2 = 17; -static int ziso_v2_block_size = 1 << 17; + +static int64_t ziso_block_number_target = -1; static int64_t ziso_max_total_blocks = ISO_ZISOFS_MAX_BLOCKS_T; static int64_t ziso_max_file_blocks = ISO_ZISOFS_MAX_BLOCKS_F; +static +int ziso_decide_v2_usage(off_t orig_size) +{ + if (ziso_v2_enabled > 1 || + (ziso_v2_enabled == 1 && orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT)) + return 1; + return 0; +} + +static +int ziso_decide_bs_log2(off_t orig_size) +{ + int bs_log2, bs_log2_min, i; + off_t bs; + + if (ziso_decide_v2_usage(orig_size)) { + bs_log2 = ziso_v2_block_size_log2; + bs_log2_min = ISO_ZISOFS_V2_MIN_LOG2; + } else { + bs_log2 = ziso_block_size_log2; + bs_log2_min = ISO_ZISOFS_V1_MIN_LOG2; + } + if (ziso_block_number_target <= 0) + return bs_log2; + + for (i = bs_log2_min; i < bs_log2; i++) { + bs = (1 << i); + if (orig_size / bs + !!(orig_size % bs) + 1 <= + ziso_block_number_target) + return i; + } + return bs_log2; +} + + +/* --------------------------- ZisofsFilterRuntime ------------------------- */ + + /* Individual runtime properties exist only as long as the stream is opened. */ typedef struct @@ -152,10 +189,10 @@ int ziso_running_destroy(ZisofsFilterRuntime **running, int flag) /* * @param flag bit0= do not set block_size, do not allocate buffers - * bit1= use ziso_v2_block_size */ static -int ziso_running_new(ZisofsFilterRuntime **running, int flag) +int ziso_running_new(ZisofsFilterRuntime **running, off_t orig_size, + int flag) { ZisofsFilterRuntime *o; *running = o = calloc(sizeof(ZisofsFilterRuntime), 1); @@ -181,10 +218,7 @@ int ziso_running_new(ZisofsFilterRuntime **running, int flag) if (flag & 1) return 1; - if (flag & 2) - o->block_size = ziso_v2_block_size; - else - o->block_size = ziso_block_size; + o->block_size = (1 << ziso_decide_bs_log2(orig_size)); #ifdef Libisofs_with_zliB o->buffer_size = compressBound((uLong) o->block_size); #else @@ -330,16 +364,6 @@ static int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired); -static -int ziso_decide_v2_usage(off_t orig_size) -{ - if (ziso_v2_enabled > 1 || - (ziso_v2_enabled == 1 && orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT)) - return 1; - return 0; -} - - /* * @param flag bit0= original stream is not open * bit1= do not destroy large @@ -366,9 +390,7 @@ int ziso_stream_close_flag(IsoStream *stream, int flag) if (cstd != NULL) { int block_size; - block_size = ziso_block_size; - if (ziso_decide_v2_usage(cstd->orig_size)) - block_size = ziso_v2_block_size; + block_size = (1 << ziso_decide_bs_log2(cstd->orig_size)); if ((!(flag & 2)) && cstd->open_counter == 1 && cstd->orig_size / block_size >= ISO_ZISOFS_MANY_BLOCKS) { if (cstd->block_pointers != NULL) { @@ -411,7 +433,8 @@ int ziso_stream_open_flag(IsoStream *stream, int flag) ZisofsFilterStreamData *data; ZisofsComprStreamData *cstd; ZisofsFilterRuntime *running = NULL; - int ret, use_v2 = 0; + int ret; + off_t orig_size = 0; if (stream == NULL) { return ISO_NULL_POINTER; @@ -426,15 +449,17 @@ int ziso_stream_open_flag(IsoStream *stream, int flag) */ stream->class->get_size(stream); } + orig_size = data->size; if (stream->class->read == &ziso_stream_compress) { cstd = (ZisofsComprStreamData *) data; cstd->open_counter++; - use_v2 = ziso_decide_v2_usage((off_t) cstd->orig_size); + orig_size = cstd->orig_size; } + if (orig_size < 0) + return ISO_ZISOFS_UNKNOWN_SIZE; - ret = ziso_running_new(&running, - (stream->class->read == &ziso_stream_uncompress) | - ((!!use_v2) << 1)); + ret = ziso_running_new(&running, orig_size, + (stream->class->read == &ziso_stream_uncompress)); if (ret < 0) { return ret; } @@ -502,7 +527,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) rng->block_buffer[8] = 0; /* @hdr_version */ rng->block_buffer[9] = 6; /* @hdr_size */ rng->block_buffer[10] = 1; /* @alg_id */ - rng->block_buffer[11] = ziso_v2_block_size_log2; + rng->block_buffer[11] = ziso_decide_bs_log2(orig_size); iso_lsb64((uint8_t *) (rng->block_buffer + 12), (uint64_t) orig_size); memset(rng->block_buffer + 20, 0, 4); @@ -516,7 +541,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) iso_lsb((unsigned char *) (rng->block_buffer + 8), (uint32_t) orig_size, 4); rng->block_buffer[12] = 4; - rng->block_buffer[13] = ziso_block_size_log2; + rng->block_buffer[13] = ziso_decide_bs_log2(orig_size); rng->block_buffer[14] = rng->block_buffer[15] = 0; rng->buffer_fill = 16; } @@ -1468,12 +1493,11 @@ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type, cnstd = stream->data; *header_size_div4 = 4; *uncompressed_size = cnstd->orig_size; + *block_size_log2 = ziso_decide_bs_log2((off_t) *uncompressed_size); if (ziso_decide_v2_usage((off_t) *uncompressed_size)) { - *block_size_log2 = ziso_v2_block_size_log2; zisofs_algo[0] = 'P'; zisofs_algo[1] = 'Z'; } else if (*uncompressed_size < (uint64_t) ISO_ZISOFS_V1_LIMIT) { - *block_size_log2 = ziso_block_size_log2; zisofs_algo[0] = 'p'; zisofs_algo[1] = 'z'; } else { @@ -1543,7 +1567,6 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag) } ziso_compression_level = params->compression_level; ziso_block_size_log2 = params->block_size_log2; - ziso_block_size = 1 << ziso_block_size_log2; if (params->version == 0) return 1; @@ -1551,11 +1574,12 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag) ziso_v2_enabled = params->v2_enabled; if (params->v2_block_size_log2 > 0) ziso_v2_block_size_log2 = params->v2_block_size_log2; - ziso_v2_block_size = 1 << ziso_v2_block_size_log2; if (params->max_total_blocks > 0) ziso_max_total_blocks = params->max_total_blocks; if (params->max_file_blocks > 0) ziso_max_file_blocks = params->max_file_blocks; + if (params->block_number_target != 0) + ziso_block_number_target = params->block_number_target; /* >>> zisofs2: more parameters */ @@ -1587,6 +1611,7 @@ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag) params->max_total_blocks = ziso_max_total_blocks; params->current_total_blocks = ziso_block_pointer_mgt((uint64_t) 0, 3); params->max_file_blocks = ziso_max_file_blocks; + params->block_number_target = ziso_block_number_target; } return 1; diff --git a/libisofs/libisofs.h b/libisofs/libisofs.h index 586384b..8e0b55b 100644 --- a/libisofs/libisofs.h +++ b/libisofs/libisofs.h @@ -8021,7 +8021,7 @@ struct iso_zisofs_ctrl { /* Set to 0 or 1 for this version of the structure * 0 = only members up to .block_size_log2 are valid - * 1 = members up to .max_file_blocks are valid + * 1 = members up to .block_number_target are valid * @since 1.5.4 */ int version; @@ -8078,6 +8078,21 @@ struct iso_zisofs_ctrl { */ uint64_t max_file_blocks; + /* + * @since 1.5.4 + * Number of block pointers of a file, which is considered low enough to + * justify a reduction of block size. If this number is > 0, then the + * lowest permissible block size is used, with which not more than the + * given number of block pointers gets produced. Upper limit is the + * setting of block size log2. + * The inavoidable end block pointer counts. E.g. a file of 55 KiB has + * 3 blocks pointers with block size log2 15, and 2 blocks pointers with + * block size log2 16. + * -1 disables this automatic block size adjustment. + * 0 keeps the current setting. + */ + int64_t block_number_target; + /* >>> ??? zisofs2: ISO_ZISOFS_MANY_BLOCKS , 0 = default 65537 */ /* >>> ??? zisofs2: a limit for number of zisofs2 files in order to keep @@ -9141,6 +9156,9 @@ int iso_conv_name_chars(IsoWriteOpts *opts, char *name, size_t name_len, /** Prevented zisofs block pointer counter underrun (WARNING,MEDIUM, -424) */ #define ISO_ZISOFS_BPT_UNDERRUN 0xD020FE58 +/** Cannot obtain size of zisofs compressed stream (FAILURE, HIGH, -425) */ +#define ISO_ZISOFS_UNKNOWN_SIZE 0xE830FE57 + /* Internal developer note: Place new error codes directly above this comment. diff --git a/libisofs/messages.c b/libisofs/messages.c index 400d12b..9b9b780 100644 --- a/libisofs/messages.c +++ b/libisofs/messages.c @@ -561,6 +561,8 @@ const char *iso_error_to_msg(int errcode) return "Too many zisofs block pointers needed overall"; case ISO_ZISOFS_BPT_UNDERRUN: return "Prevented zisofs block pointer counter underrun"; + case ISO_ZISOFS_UNKNOWN_SIZE: + return "Cannot obtain size of zisofs compressed stream"; default: return "Unknown error"; }