New iso_zisofs_ctrl parameter .block_number_target

libisofs/filters/zisofs.c

@@ -84,21 +84,58 @@ static uint64_t ziso_block_pointer_mgt(uint64_t num, int mode);
 #define ISO_ZISOFS_V2_MAX_LOG2   20
 
 
-/* --------------------------- ZisofsFilterRuntime ------------------------- */
+/* --------------------------- Runtime parameters ------------------------- */
-
 
 /* Sizes to be used for compression. Decompression learns from input header. */
 static uint8_t ziso_block_size_log2 = 15;
-static int ziso_block_size = 32768;
 
 static int ziso_v2_enabled = 0;
 static int ziso_v2_block_size_log2 = 17;
-static int ziso_v2_block_size = 1 << 17;
+
+static int64_t ziso_block_number_target = -1;
 
 static int64_t ziso_max_total_blocks = ISO_ZISOFS_MAX_BLOCKS_T;
 static int64_t ziso_max_file_blocks = ISO_ZISOFS_MAX_BLOCKS_F;
 
 
+static
+int ziso_decide_v2_usage(off_t orig_size)
+{
+    if (ziso_v2_enabled > 1 ||
+        (ziso_v2_enabled == 1 && orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT))
+        return 1;
+    return 0;
+}
+
+static
+int ziso_decide_bs_log2(off_t orig_size)
+{       
+    int bs_log2, bs_log2_min, i;
+    off_t bs;
+
+    if (ziso_decide_v2_usage(orig_size)) {
+        bs_log2 = ziso_v2_block_size_log2;
+        bs_log2_min = ISO_ZISOFS_V2_MIN_LOG2;
+    } else {
+        bs_log2 = ziso_block_size_log2;
+        bs_log2_min = ISO_ZISOFS_V1_MIN_LOG2;
+    }
+    if (ziso_block_number_target <= 0)
+        return bs_log2;
+
+    for (i = bs_log2_min; i < bs_log2; i++) {
+        bs = (1 << i);
+        if (orig_size / bs + !!(orig_size % bs) + 1 <=
+            ziso_block_number_target)
+            return i;
+    }
+    return bs_log2;
+}
+
+
+/* --------------------------- ZisofsFilterRuntime ------------------------- */
+
+
 /* Individual runtime properties exist only as long as the stream is opened.
  */
 typedef struct
@@ -152,10 +189,10 @@ int ziso_running_destroy(ZisofsFilterRuntime **running, int flag)
 
 /*
  * @param flag bit0= do not set block_size, do not allocate buffers
- *             bit1= use ziso_v2_block_size
  */
 static
-int ziso_running_new(ZisofsFilterRuntime **running, int flag)
+int ziso_running_new(ZisofsFilterRuntime **running, off_t orig_size,
+                     int flag)
 {
     ZisofsFilterRuntime *o;
     *running = o = calloc(sizeof(ZisofsFilterRuntime), 1);
@@ -181,10 +218,7 @@ int ziso_running_new(ZisofsFilterRuntime **running, int flag)
     if (flag & 1)
         return 1;
 
-    if (flag & 2)
+    o->block_size = (1 << ziso_decide_bs_log2(orig_size));
-        o->block_size = ziso_v2_block_size;
-    else
-        o->block_size = ziso_block_size;
 #ifdef Libisofs_with_zliB
     o->buffer_size = compressBound((uLong) o->block_size);
 #else
@@ -330,16 +364,6 @@ static
 int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired);
 
 
-static
-int ziso_decide_v2_usage(off_t orig_size)
-{
-    if (ziso_v2_enabled > 1 ||
-        (ziso_v2_enabled == 1 && orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT))
-        return 1;
-    return 0;
-}
-
-
 /*
  * @param flag  bit0= original stream is not open
  *              bit1= do not destroy large
@@ -366,9 +390,7 @@ int ziso_stream_close_flag(IsoStream *stream, int flag)
     if (cstd != NULL) {
         int block_size;
 
-        block_size = ziso_block_size;
+        block_size = (1 << ziso_decide_bs_log2(cstd->orig_size));
-        if (ziso_decide_v2_usage(cstd->orig_size))
-            block_size = ziso_v2_block_size;
         if ((!(flag & 2)) && cstd->open_counter == 1 &&
             cstd->orig_size / block_size >= ISO_ZISOFS_MANY_BLOCKS) {
             if (cstd->block_pointers != NULL) {
@@ -411,7 +433,8 @@ int ziso_stream_open_flag(IsoStream *stream, int flag)
     ZisofsFilterStreamData *data;
     ZisofsComprStreamData *cstd;
     ZisofsFilterRuntime *running = NULL;
-    int ret, use_v2 = 0;
+    int ret;
+    off_t orig_size = 0;
 
     if (stream == NULL) {
         return ISO_NULL_POINTER;
@@ -426,15 +449,17 @@ int ziso_stream_open_flag(IsoStream *stream, int flag)
         */
         stream->class->get_size(stream);
     }
+    orig_size = data->size;
     if (stream->class->read == &ziso_stream_compress) {
         cstd = (ZisofsComprStreamData *) data;
         cstd->open_counter++;
-        use_v2 = ziso_decide_v2_usage((off_t) cstd->orig_size);
+        orig_size = cstd->orig_size;
     }
+    if (orig_size < 0)
+        return ISO_ZISOFS_UNKNOWN_SIZE;
 
-    ret = ziso_running_new(&running,
+    ret = ziso_running_new(&running, orig_size,
-                           (stream->class->read == &ziso_stream_uncompress) |
+                           (stream->class->read == &ziso_stream_uncompress));
-                           ((!!use_v2) << 1));
     if (ret < 0) {
         return ret;
     }
@@ -502,7 +527,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
                     rng->block_buffer[8] = 0;    /* @hdr_version */
                     rng->block_buffer[9] = 6;    /* @hdr_size */
                     rng->block_buffer[10] = 1;   /* @alg_id */
-                    rng->block_buffer[11] = ziso_v2_block_size_log2;
+                    rng->block_buffer[11] = ziso_decide_bs_log2(orig_size);
                     iso_lsb64((uint8_t *) (rng->block_buffer + 12),
                               (uint64_t) orig_size);
                     memset(rng->block_buffer + 20, 0, 4);
@@ -516,7 +541,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired)
                     iso_lsb((unsigned char *) (rng->block_buffer + 8),
                             (uint32_t) orig_size, 4);
                     rng->block_buffer[12] = 4;
-                    rng->block_buffer[13] = ziso_block_size_log2;
+                    rng->block_buffer[13] = ziso_decide_bs_log2(orig_size);
                     rng->block_buffer[14] = rng->block_buffer[15] = 0;
                     rng->buffer_fill = 16;
                 }
@@ -1468,12 +1493,11 @@ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type,
         cnstd = stream->data;
         *header_size_div4 = 4;
         *uncompressed_size = cnstd->orig_size;
+        *block_size_log2 = ziso_decide_bs_log2((off_t) *uncompressed_size);
         if (ziso_decide_v2_usage((off_t) *uncompressed_size)) {
-          *block_size_log2 = ziso_v2_block_size_log2;
           zisofs_algo[0] = 'P';
           zisofs_algo[1] = 'Z';
         } else if (*uncompressed_size < (uint64_t) ISO_ZISOFS_V1_LIMIT) {
-          *block_size_log2 = ziso_block_size_log2;
           zisofs_algo[0] = 'p';
           zisofs_algo[1] = 'z';
         } else {
@@ -1543,7 +1567,6 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag)
     }
     ziso_compression_level = params->compression_level;
     ziso_block_size_log2 = params->block_size_log2;
-    ziso_block_size = 1 << ziso_block_size_log2;
 
     if (params->version == 0)
         return 1;
@@ -1551,11 +1574,12 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag)
     ziso_v2_enabled = params->v2_enabled;
     if (params->v2_block_size_log2 > 0)
         ziso_v2_block_size_log2 = params->v2_block_size_log2;
-    ziso_v2_block_size = 1 << ziso_v2_block_size_log2;
     if (params->max_total_blocks > 0)
         ziso_max_total_blocks = params->max_total_blocks;
     if (params->max_file_blocks > 0)
         ziso_max_file_blocks = params->max_file_blocks;
+    if (params->block_number_target != 0)
+        ziso_block_number_target = params->block_number_target;
 
     /* >>> zisofs2: more parameters */
 
@@ -1587,6 +1611,7 @@ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag)
         params->max_total_blocks = ziso_max_total_blocks;
         params->current_total_blocks = ziso_block_pointer_mgt((uint64_t) 0, 3);
         params->max_file_blocks = ziso_max_file_blocks;
+        params->block_number_target = ziso_block_number_target;
     }
     return 1;
 

libisofs/libisofs.h

@@ -8021,7 +8021,7 @@ struct iso_zisofs_ctrl {
 
     /* Set to 0 or 1 for this version of the structure
      * 0 = only members up to .block_size_log2 are valid
-     * 1 = members up to .max_file_blocks are valid
+     * 1 = members up to .block_number_target are valid
      *     @since 1.5.4
      */
     int version;
@@ -8078,6 +8078,21 @@ struct iso_zisofs_ctrl {
      */
     uint64_t max_file_blocks;
 
+    /*
+     * @since 1.5.4
+     * Number of block pointers of a file, which is considered low enough to
+     * justify a reduction of block size. If this number is > 0, then the
+     * lowest permissible block size is used, with which not more than the
+     * given number of block pointers gets produced. Upper limit is the
+     * setting of block size log2.
+     * The inavoidable end block pointer counts. E.g. a file of 55 KiB has
+     * 3 blocks pointers with block size log2 15, and 2 blocks pointers with
+     * block size log2 16.
+     * -1 disables this automatic block size adjustment.
+     *  0 keeps the current setting.
+     */
+    int64_t block_number_target;
+
     /* >>> ??? zisofs2: ISO_ZISOFS_MANY_BLOCKS , 0 = default 65537 */
 
     /* >>> ??? zisofs2: a limit for number of zisofs2 files in order to keep
@@ -9141,6 +9156,9 @@ int iso_conv_name_chars(IsoWriteOpts *opts, char *name, size_t name_len,
 /** Prevented zisofs block pointer counter underrun  (WARNING,MEDIUM, -424) */
 #define ISO_ZISOFS_BPT_UNDERRUN     0xD020FE58
 
+/** Cannot obtain size of zisofs compressed stream    (FAILURE, HIGH, -425) */
+#define ISO_ZISOFS_UNKNOWN_SIZE     0xE830FE57
+
 
 /* Internal developer note: 
    Place new error codes directly above this comment. 

libisofs/messages.c

@@ -561,6 +561,8 @@ const char *iso_error_to_msg(int errcode)
         return "Too many zisofs block pointers needed overall";
     case ISO_ZISOFS_BPT_UNDERRUN:
         return "Prevented zisofs block pointer counter underrun";
+    case ISO_ZISOFS_UNKNOWN_SIZE:
+        return "Cannot obtain size of zisofs compressed stream";
     default:
         return "Unknown error";
     }