libburnia/libisofs
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

New iso_zisofs_ctrl parameters bpt_discard_file_blocks , bpt_discard_free_ratio

Browse Source
This commit is contained in:
Thomas Schmitt 2020-10-22 13:22:13 +02:00
parent 239ba69925
commit cc2e0e32a3
2 changed files with 167 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
libisofs/filters/zisofs.c
View File

@ -52,6 +52,15 @@
ISO_ZISOFS_V1_LIMIT 1000000 ISO_ZISOFS_V1_LIMIT 1000000
*/ */
/* Minimum and maximum blocks sizes for version 1 and 2 */
#define ISO_ZISOFS_V1_MIN_LOG2 15
#define ISO_ZISOFS_V1_MAX_LOG2 17
#define ISO_ZISOFS_V2_MIN_LOG2 15
#define ISO_ZISOFS_V2_MAX_LOG2 20
/* ------------------- Defaults of runtime parameters ------------------ */
/* Limit for overall count of allocated block pointers: /* Limit for overall count of allocated block pointers:
2 exp 25 = 256 MiB blocklist buffer = 4 TiB uncompressed at 128 KiB 2 exp 25 = 256 MiB blocklist buffer = 4 TiB uncompressed at 128 KiB
*/ */
@ -65,23 +74,20 @@ static uint64_t ziso_block_pointer_mgt(uint64_t num, int mode);
*/ */
#define ISO_ZISOFS_MAX_BLOCKS_F 0x2000000 #define ISO_ZISOFS_MAX_BLOCKS_F 0x2000000
#ifdef Not_yeT
/* The number of blocks from which on the block pointer list shall be discarded /* The number of blocks from which on the block pointer list shall be discarded
* on iso_stream_close() of a compressing stream. This means that list and size * on iso_stream_close() of a compressing stream. This means that the pointers
* have to be determined again on next ziso_stream_get_size(). * have to be determined again on next ziso_stream_compress(), so that adding
* zisofs v1 with uint32_t pointers could at most have 131072 pointers. * a zisofs compression filter and writing the compressed stream needs in the
* Since pointers are now uint64_t, the limit tolerates half of this. * sum three read runs of the input stream.
* <= 0 disables this file size based discarding.
*/ */
#define ISO_ZISOFS_MANY_BLOCKS 65537 #define ISO_ZISOFS_MANY_BLOCKS 0
#endif /* Not_yeT */ /* A ratio describing the part of the maximum number of block pointers which
* shall be kept free by intermediate discarding of block pointers. See above
/* Minimum and maximum blocks sizes for version 1 and 2 */ * ISO_ZISOFS_MANY_BLOCKS.
#define ISO_ZISOFS_V1_MIN_LOG2 15 */
#define ISO_ZISOFS_V1_MAX_LOG2 17 #define ISO_ZISOFS_KBF_RATIO 0.5
#define ISO_ZISOFS_V2_MIN_LOG2 15
#define ISO_ZISOFS_V2_MAX_LOG2 20
/* --------------------------- Runtime parameters ------------------------- */ /* --------------------------- Runtime parameters ------------------------- */
@ -97,6 +103,9 @@ static int64_t ziso_block_number_target = -1;
static int64_t ziso_max_total_blocks = ISO_ZISOFS_MAX_BLOCKS_T; static int64_t ziso_max_total_blocks = ISO_ZISOFS_MAX_BLOCKS_T;
static int64_t ziso_max_file_blocks = ISO_ZISOFS_MAX_BLOCKS_F; static int64_t ziso_max_file_blocks = ISO_ZISOFS_MAX_BLOCKS_F;
static int64_t ziso_many_block_limit = ISO_ZISOFS_MANY_BLOCKS;
static double ziso_keep_blocks_free_ratio = ISO_ZISOFS_KBF_RATIO;
static static
int ziso_decide_v2_usage(off_t orig_size) int ziso_decide_v2_usage(off_t orig_size)
@ -328,6 +337,8 @@ typedef struct
*/ */
uint64_t block_pointer_counter; uint64_t block_pointer_counter;
uint64_t open_counter; uint64_t open_counter;
int block_pointers_dropped;
} ZisofsComprStreamData; } ZisofsComprStreamData;
@ -374,6 +385,8 @@ int ziso_stream_close_flag(IsoStream *stream, int flag)
{ {
ZisofsFilterStreamData *data; ZisofsFilterStreamData *data;
ZisofsComprStreamData *cstd = NULL; ZisofsComprStreamData *cstd = NULL;
int block_size, discard;
double max_blocks, free_blocks;
if (stream == NULL) { if (stream == NULL) {
return ISO_NULL_POINTER; return ISO_NULL_POINTER;
@ -382,28 +395,36 @@ int ziso_stream_close_flag(IsoStream *stream, int flag)
if (stream->class->read == &ziso_stream_compress) if (stream->class->read == &ziso_stream_compress)
cstd = (ZisofsComprStreamData *) data; cstd = (ZisofsComprStreamData *) data;
#ifdef Not_yeT
/* >>> zisofs2:
research whether zisofs streams get opened and closed often */
if (cstd != NULL) { if (cstd != NULL) {
int block_size;
block_size = (1 << ziso_decide_bs_log2(cstd->orig_size)); block_size = (1 << ziso_decide_bs_log2(cstd->orig_size));
if ((!(flag & 2)) && cstd->open_counter == 1 && max_blocks = ziso_max_file_blocks;
cstd->orig_size / block_size >= ISO_ZISOFS_MANY_BLOCKS) { if (max_blocks < 1.0)
if (cstd->block_pointers != NULL) { max_blocks = 1.0;
ziso_block_pointer_mgt(cstd->block_pointer_counter, 2); free_blocks = ziso_max_total_blocks -
free((char *) cstd->block_pointers); ziso_block_pointer_mgt((uint64_t) 0, 3);
} discard = 1;
if (flag & 2) {
discard = 0;
} else if (cstd->block_pointers == NULL) {
discard = 0;
} else if (cstd->open_counter != 1) {
discard = 0;
} else if (ziso_many_block_limit <= 0 ||
cstd->orig_size / block_size + !!(cstd->orig_size % block_size)
+ 1 < (uint64_t) ziso_many_block_limit) {
if (ziso_keep_blocks_free_ratio < 0.0 ||
free_blocks / max_blocks >= ziso_keep_blocks_free_ratio)
discard = 0;
}
if (discard) {
ziso_block_pointer_mgt(cstd->block_pointer_counter, 2);
free((char *) cstd->block_pointers);
cstd->block_pointers_dropped = 1;
cstd->block_pointers = NULL; cstd->block_pointers = NULL;
data->size = -1; cstd->block_pointer_counter = 0;
} }
} }
#endif /* Not_yeT */
if (data->running == NULL) { if (data->running == NULL) {
return 1; return 1;
} }
@ -480,6 +501,52 @@ int ziso_stream_open(IsoStream *stream)
} }
/* @param flag bit0= stream is already open
bit1= close stream with flag bit1
*/
static
off_t ziso_stream_measure_size(IsoStream *stream, int flag)
{
int ret, ret_close;
off_t count = 0;
ZisofsFilterStreamData *data;
char buf[64 * 1024];
size_t bufsize = 64 * 1024;
if (stream == NULL)
return ISO_NULL_POINTER;
data = stream->data;
/* Run filter command and count output bytes */
if (!(flag & 1)) {
ret = ziso_stream_open_flag(stream, 1);
if (ret < 0)
return ret;
}
if (stream->class->read == &ziso_stream_uncompress) {
/* It is enough to read the header part of a compressed file */
ret = ziso_stream_uncompress(stream, buf, 0);
count = data->size;
} else {
/* The size of the compression result has to be counted */
while (1) {
ret = stream->class->read(stream, buf, bufsize);
if (ret <= 0)
break;
count += ret;
}
}
ret_close = ziso_stream_close_flag(stream, flag & 2);
if (ret < 0)
return ret;
if (ret_close < 0)
return ret_close;
data->size = count;
return count;
}
static static
int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
{ {
@ -490,7 +557,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
ZisofsComprStreamData *data; ZisofsComprStreamData *data;
ZisofsFilterRuntime *rng; ZisofsFilterRuntime *rng;
size_t fill = 0; size_t fill = 0;
off_t orig_size, next_pt; off_t orig_size, next_pt, measure_ret;
char *cbuf = buf; char *cbuf = buf;
uLongf buf_len; uLongf buf_len;
uint64_t *copy_base, num_blocks = 0; uint64_t *copy_base, num_blocks = 0;
@ -507,6 +574,23 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
return rng->error_ret; return rng->error_ret;
} }
if (data->block_pointers_dropped) {
/* The list was dropped after measurement of compressed size. But this
* run of the function expects it as already filled with pointer
* values. So now they have to be re-computed by extra runs of this
* function in the course of compressed size measurement.
*/
data->block_pointers_dropped = 0;
measure_ret = ziso_stream_measure_size(stream, 1 | 2);
if (measure_ret < 0)
return (rng->error_ret = measure_ret);
/* Stream was closed. Open it again, without any size determination. */
ret = ziso_stream_open_flag(stream, 1);
if (ret < 0)
return ret;
}
while (1) { while (1) {
if (rng->state == 0) { if (rng->state == 0) {
/* Delivering file header */ /* Delivering file header */
@ -554,11 +638,14 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
if (rng->state == 1) { if (rng->state == 1) {
/* Delivering block pointers */; /* Delivering block pointers */;
if (rng->block_pointer_fill == 0) { if (rng->block_pointer_fill == 0 || data->block_pointers == NULL) {
/* Initialize block pointer writing */ /* Initialize block pointer writing */
rng->block_pointer_rpos = 0; rng->block_pointer_rpos = 0;
num_blocks = data->orig_size / rng->block_size num_blocks = data->orig_size / rng->block_size
+ 1 + !!(data->orig_size % rng->block_size); + 1 + !!(data->orig_size % rng->block_size);
if (rng->block_pointer_fill > 0 &&
(int64_t) num_blocks != rng->block_pointer_fill)
return (rng->error_ret = ISO_FILTER_WRONG_INPUT);
rng->block_pointer_fill = num_blocks; rng->block_pointer_fill = num_blocks;
if (data->block_pointers == NULL) { if (data->block_pointers == NULL) {
/* On the first pass, create pointer array with all 0s */ /* On the first pass, create pointer array with all 0s */
@ -1007,47 +1094,16 @@ int ziso_stream_uncompress(IsoStream *stream, void *buf, size_t desired)
static static
off_t ziso_stream_get_size(IsoStream *stream) off_t ziso_stream_get_size(IsoStream *stream)
{ {
int ret, ret_close; off_t ret;
off_t count = 0;
ZisofsFilterStreamData *data; ZisofsFilterStreamData *data;
char buf[64 * 1024];
size_t bufsize = 64 * 1024;
if (stream == NULL) { if (stream == NULL)
return ISO_NULL_POINTER; return ISO_NULL_POINTER;
}
data = stream->data; data = stream->data;
if (data->size >= 0)
if (data->size >= 0) {
return data->size; return data->size;
} ret = ziso_stream_measure_size(stream, 0);
return ret;
/* Run filter command and count output bytes */
ret = ziso_stream_open_flag(stream, 1);
if (ret < 0) {
return ret;
}
if (stream->class->read == &ziso_stream_uncompress) {
/* It is enough to read the header part of a compressed file */
ret = ziso_stream_uncompress(stream, buf, 0);
count = data->size;
} else {
/* The size of the compression result has to be counted */
while (1) {
ret = stream->class->read(stream, buf, bufsize);
if (ret <= 0)
break;
count += ret;
}
}
ret_close = ziso_stream_close_flag(stream, 2);
if (ret < 0)
return ret;
if (ret_close < 0)
return ret_close;
data->size = count;
return count;
} }
@ -1158,6 +1214,11 @@ int ziso_clone_stream(IsoStream *old_stream, IsoStream **new_stream, int flag)
compr->block_pointers = NULL; compr->block_pointers = NULL;
compr->block_pointer_counter = 0; compr->block_pointer_counter = 0;
compr->open_counter = 0; compr->open_counter = 0;
if (old_compr->block_pointers != NULL ||
old_compr->block_pointers_dropped)
compr->block_pointers_dropped = 1;
else
compr->block_pointers_dropped = 0;
} }
old_stream_data = (ZisofsFilterStreamData *) old_stream->data; old_stream_data = (ZisofsFilterStreamData *) old_stream->data;
stream_data->orig = new_input_stream; stream_data->orig = new_input_stream;
@ -1322,6 +1383,7 @@ int ziso_filter_get_filter(FilterContext *filter, IsoStream *original,
cnstd->block_pointers = NULL; cnstd->block_pointers = NULL;
cnstd->block_pointer_counter = 0; cnstd->block_pointer_counter = 0;
cnstd->open_counter = 0; cnstd->open_counter = 0;
cnstd->block_pointers_dropped = 0;
str->class = &ziso_stream_compress_class; str->class = &ziso_stream_compress_class;
ziso_ref_count++; ziso_ref_count++;
} }
@ -1606,6 +1668,10 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag)
ziso_max_file_blocks = params->max_file_blocks; ziso_max_file_blocks = params->max_file_blocks;
if (params->block_number_target != 0) if (params->block_number_target != 0)
ziso_block_number_target = params->block_number_target; ziso_block_number_target = params->block_number_target;
if (params->bpt_discard_file_blocks != 0)
ziso_many_block_limit = params->bpt_discard_file_blocks;
if (params->bpt_discard_free_ratio != 0.0)
ziso_keep_blocks_free_ratio = params->bpt_discard_free_ratio;
/* >>> zisofs2: more parameters */ /* >>> zisofs2: more parameters */
@ -1638,6 +1704,8 @@ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag)
params->current_total_blocks = ziso_block_pointer_mgt((uint64_t) 0, 3); params->current_total_blocks = ziso_block_pointer_mgt((uint64_t) 0, 3);
params->max_file_blocks = ziso_max_file_blocks; params->max_file_blocks = ziso_max_file_blocks;
params->block_number_target = ziso_block_number_target; params->block_number_target = ziso_block_number_target;
params->bpt_discard_file_blocks = ziso_many_block_limit;
params->bpt_discard_free_ratio = ziso_keep_blocks_free_ratio;
} }
return 1; return 1;

39
libisofs/libisofs.h
View File

@ -8021,7 +8021,7 @@ struct iso_zisofs_ctrl {
/* Set to 0 or 1 for this version of the structure /* Set to 0 or 1 for this version of the structure
* 0 = only members up to .block_size_log2 are valid * 0 = only members up to .block_size_log2 are valid
* 1 = members up to .block_number_target are valid * 1 = members up to .bpt_discard_free_ratio are valid
* @since 1.5.4 * @since 1.5.4
*/ */
int version; int version;
@ -8042,7 +8042,6 @@ struct iso_zisofs_ctrl {
/* ------------------- Only valid with .version >= 1 ------------------- */ /* ------------------- Only valid with .version >= 1 ------------------- */
/* /*
* @since 1.5.4
* Whether to produce zisofs2 (zisofs version 2) file headers and ZF * Whether to produce zisofs2 (zisofs version 2) file headers and ZF
* entries for files which get compressed: * entries for files which get compressed:
* 0 = do not produce zisofs2, * 0 = do not produce zisofs2,
@ -8050,36 +8049,36 @@ struct iso_zisofs_ctrl {
* This is the default. * This is the default.
* 1 = zisofs2 is enabled for file size 4 GiB or more * 1 = zisofs2 is enabled for file size 4 GiB or more
* 2 = zisofs2 shall be used if zisofs is used at all * 2 = zisofs2 shall be used if zisofs is used at all
* @since 1.5.4
*/ */
int v2_enabled; int v2_enabled;
/* /*
* @since 1.5.4
* Log2 of block size for zisofs2 files. 0 keeps current setting. * Log2 of block size for zisofs2 files. 0 keeps current setting.
* Allowed are 15 = 32 kiB to 20 = 1024 kiB. * Allowed are 15 = 32 kiB to 20 = 1024 kiB.
* @since 1.5.4
*/ */
uint8_t v2_block_size_log2; uint8_t v2_block_size_log2;
/* /*
* @since 1.5.4
* Maximum overall number of blocklist pointers. 0 keeps current setting. * Maximum overall number of blocklist pointers. 0 keeps current setting.
* @since 1.5.4
*/ */
uint64_t max_total_blocks; uint64_t max_total_blocks;
/* /*
* @since 1.5.4
* Ignored as input value: Number of allocated zisofs block pointers. * Ignored as input value: Number of allocated zisofs block pointers.
* @since 1.5.4
*/ */
uint64_t current_total_blocks; uint64_t current_total_blocks;
/* /*
* @since 1.5.4
* Maximum number of blocklist pointers per file. 0 keeps current setting. * Maximum number of blocklist pointers per file. 0 keeps current setting.
* @since 1.5.4
*/ */
uint64_t max_file_blocks; uint64_t max_file_blocks;
/* /*
* @since 1.5.4
* Number of block pointers of a file, which is considered low enough to * Number of block pointers of a file, which is considered low enough to
* justify a reduction of block size. If this number is > 0, then the * justify a reduction of block size. If this number is > 0, then the
* lowest permissible block size is used, with which not more than the * lowest permissible block size is used, with which not more than the
@ -8090,10 +8089,34 @@ struct iso_zisofs_ctrl {
* block size log2 16. * block size log2 16.
* -1 disables this automatic block size adjustment. * -1 disables this automatic block size adjustment.
* 0 keeps the current setting. * 0 keeps the current setting.
* @since 1.5.4
*/ */
int64_t block_number_target; int64_t block_number_target;
/* >>> ??? zisofs2: ISO_ZISOFS_MANY_BLOCKS , 0 = default 65537 */ /*
* The number of blocks from which on the block pointer list shall be
* discarded on iso_stream_close() of a compressing stream. This means that
* the pointers have to be determined again on next ziso_stream_compress(),
* so that adding a zisofs compression filter and writing the compressed
* stream needs in the sum three read runs of the input stream.
* 0 keeps the current setting.
* < 0 disables this file size based discarding.
* @since 1.5.4
*/
int64_t bpt_discard_file_blocks;
/*
* A ratio describing the part of max_file_blocks which shall be kept free
* by intermediate discarding of block pointers.
* See above bpt_discard_file_blocks .
* It makes sense to set this to 1.0 if max_file_blocks is substantially
* smaller than max_total_blocks.
* 0.0 keeps the current setting.
* < 0.0 disables this memory consumption based discarding.
* @since 1.5.4
*/
double bpt_discard_free_ratio;
/* >>> ??? zisofs2: a limit for number of zisofs2 files in order to keep /* >>> ??? zisofs2: a limit for number of zisofs2 files in order to keep
the number of these old kernel warnings bearable: the number of these old kernel warnings bearable: