New iso_zisofs_ctrl parameter .block_number_target

This commit is contained in:
Thomas Schmitt 2020-10-17 14:40:24 +02:00
parent f291e37ec1
commit 2ca3b292fb
3 changed files with 80 additions and 35 deletions

View File

@ -84,21 +84,58 @@ static uint64_t ziso_block_pointer_mgt(uint64_t num, int mode);
#define ISO_ZISOFS_V2_MAX_LOG2 20 #define ISO_ZISOFS_V2_MAX_LOG2 20
/* --------------------------- ZisofsFilterRuntime ------------------------- */ /* --------------------------- Runtime parameters ------------------------- */
/* Sizes to be used for compression. Decompression learns from input header. */ /* Sizes to be used for compression. Decompression learns from input header. */
static uint8_t ziso_block_size_log2 = 15; static uint8_t ziso_block_size_log2 = 15;
static int ziso_block_size = 32768;
static int ziso_v2_enabled = 0; static int ziso_v2_enabled = 0;
static int ziso_v2_block_size_log2 = 17; static int ziso_v2_block_size_log2 = 17;
static int ziso_v2_block_size = 1 << 17;
static int64_t ziso_block_number_target = -1;
static int64_t ziso_max_total_blocks = ISO_ZISOFS_MAX_BLOCKS_T; static int64_t ziso_max_total_blocks = ISO_ZISOFS_MAX_BLOCKS_T;
static int64_t ziso_max_file_blocks = ISO_ZISOFS_MAX_BLOCKS_F; static int64_t ziso_max_file_blocks = ISO_ZISOFS_MAX_BLOCKS_F;
static
int ziso_decide_v2_usage(off_t orig_size)
{
if (ziso_v2_enabled > 1 ||
(ziso_v2_enabled == 1 && orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT))
return 1;
return 0;
}
static
int ziso_decide_bs_log2(off_t orig_size)
{
int bs_log2, bs_log2_min, i;
off_t bs;
if (ziso_decide_v2_usage(orig_size)) {
bs_log2 = ziso_v2_block_size_log2;
bs_log2_min = ISO_ZISOFS_V2_MIN_LOG2;
} else {
bs_log2 = ziso_block_size_log2;
bs_log2_min = ISO_ZISOFS_V1_MIN_LOG2;
}
if (ziso_block_number_target <= 0)
return bs_log2;
for (i = bs_log2_min; i < bs_log2; i++) {
bs = (1 << i);
if (orig_size / bs + !!(orig_size % bs) + 1 <=
ziso_block_number_target)
return i;
}
return bs_log2;
}
/* --------------------------- ZisofsFilterRuntime ------------------------- */
/* Individual runtime properties exist only as long as the stream is opened. /* Individual runtime properties exist only as long as the stream is opened.
*/ */
typedef struct typedef struct
@ -152,10 +189,10 @@ int ziso_running_destroy(ZisofsFilterRuntime **running, int flag)
/* /*
* @param flag bit0= do not set block_size, do not allocate buffers * @param flag bit0= do not set block_size, do not allocate buffers
* bit1= use ziso_v2_block_size
*/ */
static static
int ziso_running_new(ZisofsFilterRuntime **running, int flag) int ziso_running_new(ZisofsFilterRuntime **running, off_t orig_size,
int flag)
{ {
ZisofsFilterRuntime *o; ZisofsFilterRuntime *o;
*running = o = calloc(sizeof(ZisofsFilterRuntime), 1); *running = o = calloc(sizeof(ZisofsFilterRuntime), 1);
@ -181,10 +218,7 @@ int ziso_running_new(ZisofsFilterRuntime **running, int flag)
if (flag & 1) if (flag & 1)
return 1; return 1;
if (flag & 2) o->block_size = (1 << ziso_decide_bs_log2(orig_size));
o->block_size = ziso_v2_block_size;
else
o->block_size = ziso_block_size;
#ifdef Libisofs_with_zliB #ifdef Libisofs_with_zliB
o->buffer_size = compressBound((uLong) o->block_size); o->buffer_size = compressBound((uLong) o->block_size);
#else #else
@ -330,16 +364,6 @@ static
int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired); int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired);
static
int ziso_decide_v2_usage(off_t orig_size)
{
if (ziso_v2_enabled > 1 ||
(ziso_v2_enabled == 1 && orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT))
return 1;
return 0;
}
/* /*
* @param flag bit0= original stream is not open * @param flag bit0= original stream is not open
* bit1= do not destroy large * bit1= do not destroy large
@ -366,9 +390,7 @@ int ziso_stream_close_flag(IsoStream *stream, int flag)
if (cstd != NULL) { if (cstd != NULL) {
int block_size; int block_size;
block_size = ziso_block_size; block_size = (1 << ziso_decide_bs_log2(cstd->orig_size));
if (ziso_decide_v2_usage(cstd->orig_size))
block_size = ziso_v2_block_size;
if ((!(flag & 2)) && cstd->open_counter == 1 && if ((!(flag & 2)) && cstd->open_counter == 1 &&
cstd->orig_size / block_size >= ISO_ZISOFS_MANY_BLOCKS) { cstd->orig_size / block_size >= ISO_ZISOFS_MANY_BLOCKS) {
if (cstd->block_pointers != NULL) { if (cstd->block_pointers != NULL) {
@ -411,7 +433,8 @@ int ziso_stream_open_flag(IsoStream *stream, int flag)
ZisofsFilterStreamData *data; ZisofsFilterStreamData *data;
ZisofsComprStreamData *cstd; ZisofsComprStreamData *cstd;
ZisofsFilterRuntime *running = NULL; ZisofsFilterRuntime *running = NULL;
int ret, use_v2 = 0; int ret;
off_t orig_size = 0;
if (stream == NULL) { if (stream == NULL) {
return ISO_NULL_POINTER; return ISO_NULL_POINTER;
@ -426,15 +449,17 @@ int ziso_stream_open_flag(IsoStream *stream, int flag)
*/ */
stream->class->get_size(stream); stream->class->get_size(stream);
} }
orig_size = data->size;
if (stream->class->read == &ziso_stream_compress) { if (stream->class->read == &ziso_stream_compress) {
cstd = (ZisofsComprStreamData *) data; cstd = (ZisofsComprStreamData *) data;
cstd->open_counter++; cstd->open_counter++;
use_v2 = ziso_decide_v2_usage((off_t) cstd->orig_size); orig_size = cstd->orig_size;
} }
if (orig_size < 0)
return ISO_ZISOFS_UNKNOWN_SIZE;
ret = ziso_running_new(&running, ret = ziso_running_new(&running, orig_size,
(stream->class->read == &ziso_stream_uncompress) | (stream->class->read == &ziso_stream_uncompress));
((!!use_v2) << 1));
if (ret < 0) { if (ret < 0) {
return ret; return ret;
} }
@ -502,7 +527,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
rng->block_buffer[8] = 0; /* @hdr_version */ rng->block_buffer[8] = 0; /* @hdr_version */
rng->block_buffer[9] = 6; /* @hdr_size */ rng->block_buffer[9] = 6; /* @hdr_size */
rng->block_buffer[10] = 1; /* @alg_id */ rng->block_buffer[10] = 1; /* @alg_id */
rng->block_buffer[11] = ziso_v2_block_size_log2; rng->block_buffer[11] = ziso_decide_bs_log2(orig_size);
iso_lsb64((uint8_t *) (rng->block_buffer + 12), iso_lsb64((uint8_t *) (rng->block_buffer + 12),
(uint64_t) orig_size); (uint64_t) orig_size);
memset(rng->block_buffer + 20, 0, 4); memset(rng->block_buffer + 20, 0, 4);
@ -516,7 +541,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
iso_lsb((unsigned char *) (rng->block_buffer + 8), iso_lsb((unsigned char *) (rng->block_buffer + 8),
(uint32_t) orig_size, 4); (uint32_t) orig_size, 4);
rng->block_buffer[12] = 4; rng->block_buffer[12] = 4;
rng->block_buffer[13] = ziso_block_size_log2; rng->block_buffer[13] = ziso_decide_bs_log2(orig_size);
rng->block_buffer[14] = rng->block_buffer[15] = 0; rng->block_buffer[14] = rng->block_buffer[15] = 0;
rng->buffer_fill = 16; rng->buffer_fill = 16;
} }
@ -1468,12 +1493,11 @@ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type,
cnstd = stream->data; cnstd = stream->data;
*header_size_div4 = 4; *header_size_div4 = 4;
*uncompressed_size = cnstd->orig_size; *uncompressed_size = cnstd->orig_size;
*block_size_log2 = ziso_decide_bs_log2((off_t) *uncompressed_size);
if (ziso_decide_v2_usage((off_t) *uncompressed_size)) { if (ziso_decide_v2_usage((off_t) *uncompressed_size)) {
*block_size_log2 = ziso_v2_block_size_log2;
zisofs_algo[0] = 'P'; zisofs_algo[0] = 'P';
zisofs_algo[1] = 'Z'; zisofs_algo[1] = 'Z';
} else if (*uncompressed_size < (uint64_t) ISO_ZISOFS_V1_LIMIT) { } else if (*uncompressed_size < (uint64_t) ISO_ZISOFS_V1_LIMIT) {
*block_size_log2 = ziso_block_size_log2;
zisofs_algo[0] = 'p'; zisofs_algo[0] = 'p';
zisofs_algo[1] = 'z'; zisofs_algo[1] = 'z';
} else { } else {
@ -1543,7 +1567,6 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag)
} }
ziso_compression_level = params->compression_level; ziso_compression_level = params->compression_level;
ziso_block_size_log2 = params->block_size_log2; ziso_block_size_log2 = params->block_size_log2;
ziso_block_size = 1 << ziso_block_size_log2;
if (params->version == 0) if (params->version == 0)
return 1; return 1;
@ -1551,11 +1574,12 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag)
ziso_v2_enabled = params->v2_enabled; ziso_v2_enabled = params->v2_enabled;
if (params->v2_block_size_log2 > 0) if (params->v2_block_size_log2 > 0)
ziso_v2_block_size_log2 = params->v2_block_size_log2; ziso_v2_block_size_log2 = params->v2_block_size_log2;
ziso_v2_block_size = 1 << ziso_v2_block_size_log2;
if (params->max_total_blocks > 0) if (params->max_total_blocks > 0)
ziso_max_total_blocks = params->max_total_blocks; ziso_max_total_blocks = params->max_total_blocks;
if (params->max_file_blocks > 0) if (params->max_file_blocks > 0)
ziso_max_file_blocks = params->max_file_blocks; ziso_max_file_blocks = params->max_file_blocks;
if (params->block_number_target != 0)
ziso_block_number_target = params->block_number_target;
/* >>> zisofs2: more parameters */ /* >>> zisofs2: more parameters */
@ -1587,6 +1611,7 @@ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag)
params->max_total_blocks = ziso_max_total_blocks; params->max_total_blocks = ziso_max_total_blocks;
params->current_total_blocks = ziso_block_pointer_mgt((uint64_t) 0, 3); params->current_total_blocks = ziso_block_pointer_mgt((uint64_t) 0, 3);
params->max_file_blocks = ziso_max_file_blocks; params->max_file_blocks = ziso_max_file_blocks;
params->block_number_target = ziso_block_number_target;
} }
return 1; return 1;

View File

@ -8021,7 +8021,7 @@ struct iso_zisofs_ctrl {
/* Set to 0 or 1 for this version of the structure /* Set to 0 or 1 for this version of the structure
* 0 = only members up to .block_size_log2 are valid * 0 = only members up to .block_size_log2 are valid
* 1 = members up to .max_file_blocks are valid * 1 = members up to .block_number_target are valid
* @since 1.5.4 * @since 1.5.4
*/ */
int version; int version;
@ -8078,6 +8078,21 @@ struct iso_zisofs_ctrl {
*/ */
uint64_t max_file_blocks; uint64_t max_file_blocks;
/*
* @since 1.5.4
* Number of block pointers of a file, which is considered low enough to
* justify a reduction of block size. If this number is > 0, then the
* lowest permissible block size is used, with which not more than the
* given number of block pointers gets produced. Upper limit is the
* setting of block size log2.
* The inavoidable end block pointer counts. E.g. a file of 55 KiB has
* 3 blocks pointers with block size log2 15, and 2 blocks pointers with
* block size log2 16.
* -1 disables this automatic block size adjustment.
* 0 keeps the current setting.
*/
int64_t block_number_target;
/* >>> ??? zisofs2: ISO_ZISOFS_MANY_BLOCKS , 0 = default 65537 */ /* >>> ??? zisofs2: ISO_ZISOFS_MANY_BLOCKS , 0 = default 65537 */
/* >>> ??? zisofs2: a limit for number of zisofs2 files in order to keep /* >>> ??? zisofs2: a limit for number of zisofs2 files in order to keep
@ -9141,6 +9156,9 @@ int iso_conv_name_chars(IsoWriteOpts *opts, char *name, size_t name_len,
/** Prevented zisofs block pointer counter underrun (WARNING,MEDIUM, -424) */ /** Prevented zisofs block pointer counter underrun (WARNING,MEDIUM, -424) */
#define ISO_ZISOFS_BPT_UNDERRUN 0xD020FE58 #define ISO_ZISOFS_BPT_UNDERRUN 0xD020FE58
/** Cannot obtain size of zisofs compressed stream (FAILURE, HIGH, -425) */
#define ISO_ZISOFS_UNKNOWN_SIZE 0xE830FE57
/* Internal developer note: /* Internal developer note:
Place new error codes directly above this comment. Place new error codes directly above this comment.

View File

@ -561,6 +561,8 @@ const char *iso_error_to_msg(int errcode)
return "Too many zisofs block pointers needed overall"; return "Too many zisofs block pointers needed overall";
case ISO_ZISOFS_BPT_UNDERRUN: case ISO_ZISOFS_BPT_UNDERRUN:
return "Prevented zisofs block pointer counter underrun"; return "Prevented zisofs block pointer counter underrun";
case ISO_ZISOFS_UNKNOWN_SIZE:
return "Cannot obtain size of zisofs compressed stream";
default: default:
return "Unknown error"; return "Unknown error";
} }