diff --git a/doc/zisofs2_format.txt b/doc/zisofs2_format.txt new file mode 100644 index 0000000..fccbfb1 --- /dev/null +++ b/doc/zisofs2_format.txt @@ -0,0 +1,239 @@ + + Description of the zisofs2 Format + Revision 2.0-dev + + as of zisofs2-tools by + Valentín KIVACHUK BURDÁ and Thomas SCHMITT + + 1 Oct 2020 + + +The zisofs2 format was invented by Valentín KIVACHUK BURDÁ and +Thomas SCHMITT (as extension of zisofs by H. Peter Anvin). It compresses +data file content, marks it by a header and provides a pointer array for +coarse random access. Within a RRIP enhanced ISO 9660 image the format +is additionally marked by a System Use entry with signature "ZF". + +The uncompressed size of a single zisofs2 compressed file is restricted +to 2^64 - 1 bytes. Larger files shall not be compressed. + +The format of version 1 of zisofs is supported by this specification. +Using it for files with uncompressed size smaller than 4 GiB is friendly +towards software which does not know about zisofs2. +See section **LEGACY** for a summary of version 1 of zisofs. + + + + Data Types + +ISO 9660:7.3.1 - little endian 4-byte words +ISO 9660:7.1.1 - unsigned single bytes +ISO 9660:7.3.3 - 8-bytes value, first in little endian, then big endian. +#uint64 - 8-bytes unsigned value in little endian + + Supported compressors + +The file header has this layout: + @alg_id @alg_char Description + 1 'PZ' (50)(5A) Zlib + 2 'XZ' (78)(7A) XZ + 3 'L4' (6C)(34) LZ4 + 4 'ZD' (7A)(64) Zstandard + 5 'B2' (62)(32) Bzip2 + +@alg_id is a 7.1.1 value. @alg_char is 2 ASCII characters stored as 2 bytes +Values of @alg_id = 0 and @alg_char = 'pz'(70)(7A) are reserved and +must not be used. Other compressors are allowed and may be added to this +list in the future + + Compressor strategy + +The default strategy for a compressor is to compress each input data block +independently. The zisofs2 spec may define in the future other strategies, +which will have a new @alg_id, @alg_char and a description in this section. + + + File Header + +The file header has this layout: +Offset Type Identifier Contents +-------------------------------------------------------------------------- + 0 (8 bytes) @hdr_magic Magic num (EF 22 55 A1 BC 1B 95 A0) + 8 7.1.1 @hdr_version File header version (0) + 9 7.1.1 @hdr_size header_size >> 2 (6) + 10 7.1.1 @alg_id Algorithm Type (>=1) + 11 7.1.1 @hdr_bsize log2(block_size) (15, 16, or 17) + 12 #uint64 @size Uncompressed file size + 20 (4 bytes) - Padding. Ignored + +So its size is 24. + +Readers shall be able to handle log2(block_size) values 15, 16 and 17 +i.e. block sizes 32 kB, 64 kB, and 128 kB. Writers must not use +other sizes. + + Block Pointers + +There are ceil(input_size / block_size) input resp. output blocks. +Each input block is of fixed size whereas the output blocks have varying +size (down to 0). For each output block there is an offset pointer giving +its byte address in the overall file content. The next block pointer in the +array tells the start of the next block which begins immediately after the +end of its predecessor. A final pointer (*eob*) gives the first invalid +byte address and thus marks the end of the last block. + +So there are ceil(input_size / block_size) + 1 block pointers. +They are stored directly after the file header, i.e. beginning at byte 24, +as an array of values in #uint64 format (8 bytes). + +Legacy format (zisofs) may be used, which is described in section *LEGACY* + + + + Data Part + +The data part begins immediately after the pointer array (*eob*). In +principle it consists of the variable length output blocks as delivered by +different compression algorithms when fed with the fixed size input blocks. + +A special case of input and output block is defined: +Zero-length blocks represent a block full of 0-bytes. +Such input blocks do not get processed by compress2() but shall be mapped +to 0-sized output directly. Vice versa 0-sized blocks have to bypass +uncompress() when being read. + + + ZF System Use Entry Format + +The ZF entry follows the general layout of SUSP resp. RRIP. +Its fields are: + + [1] "BP 1 to BP 2 - Signature Word" shall be (5A)(46) ("ZF"). + + [2] "BP 3 - Length" shall specify as an 8-bit number the length in + bytes of the ZF entry recorded according to ISO 9660:7.1.1. + This length is 16 decimal. + Refer to **LEGACY** + + [3] "BP 4 - System Use Entry Version" shall be 2 as in ISO 9660:7.1.1. + Refer to **LEGACY** + + [4] "BP 5 to BP 6 - Algorithm" shall be two chars to indicate the + compression algorithm. For example, (50)(5A) ("PZ") + (This is a copy of @alg_char). Refer to **LEGACY** + + [5] "BP 7 - Header Size Div 4" shall specify as an 8-bit number the + number of 4-byte words in the header part of the file data recorded + according to ISO 9660:7.1.1. + (This is a copy of @hdr_size). + + [6] "BP 8 - Log2 of Block Size" shall specify as an 8-bit number the + binary logarithm of the compression block size recorded according to + ISO 9660:7.1.1. + (This is a copy of header byte 13 (@hdr_bsize), resp. header BP 14. + The value has to be 15, 16 or 17 i.e. 32 kiB, 64 kiB, or 128 kiB.) + + [7] "BP 9 to BP 16 - Virtual Uncompressed File Size" shall contain + as a 64-bit unsigned little endian number the uncompressed + file size represented by the given extent. Refer to **LEGACY** + + + | 'Z' | 'F' | LENGTH | 2 | 'P' | 'Z' | HEADER SIZE DIV 4 | + | LOG2 BLOCK SIZE | UNCOMPRESSED SIZE | + +Example (block size 128 kiB, uncompressed file size = 40 TB): + { 'Z', 'F', 16, 2, 'P', 'Z', 8, 17, + 0x00, 0x80, 0xCA, 0x39, 0x61, 0x24, 0x00, 0x00 } + + + + **LEGACY** + +zisofs2 supports old readers by respecting the zisofs format. This section +describes which definitions from zisofs2 must change to be compatible +with zisofs. + +- General behaviour + The uncompressed size of a single zisofs compressed file is restricted + to 4 GiB - 1. Larger files shall not be compressed. + +- Supported algorithms + Only algorithm Zlib with default strategy is supported. + +- The file header must follow this structure: + + Offset Type Identifier Contents + 0 (8 bytes) @hdr_magic Magic number (37 E4 53 96 C9 DB D6 07) + 8 7.3.1 @size Uncompressed file size + 12 7.1.1 @hdr_size header_size >> 2 (4) + 13 7.1.1 @hdr_bsize log2(block_size) (15, 16, or 17) + 14 (2 bytes) - Reserved, must be zero + + So its size is 16. + +- Block pointers + The array must use ISO 9660:7.3.1 (4 bytes) values. + +- ZF entry + +Its fields are: + + [1] "BP 1 to BP 2 - Signature Word" shall be (5A)(46) ("ZF"). + + [2] "BP 3 - Length" must be 16 decimal. + + [3] "BP 4 - System Use Entry Version" must be 1. + + [4] "BP 5 to BP 6 - Algorithm" must be (70)(7A) ("pz"). + + [5] "BP 7 - Header Size Div 4" - same as zisofs2. + + [6] "BP 8 - Log2 of Block Size" - same as zisofs2. + + [7] "BP 9 to BP 16 - Uncompressed Size" This field shall be recorded + according to ISO 9660:7.3.3. + (This number is the same as @size ) + + | 'Z' | 'F' | LENGTH | 1 | 'p' | 'z' | HEADER SIZE DIV 4 | + | LOG2 BLOCK SIZE | UNCOMPRESSED SIZE | + +Example (block size 32 kiB, uncompressed file size = 1,234,567 bytes): + { 'Z', 'F', 16, 1, 'p', 'z', 4, 15, + 0x87, 0xD6, 0x12, 0x00, 0x00, 0x12, 0xD6, 0x87 } + + +References: + +zisofs2-tools + https://github.com/vk496/zisofs2-tools + +zisofs-tools + http://freshmeat.net/projects/zisofs-tools/ + +zlib: + /usr/include/zlib.h + +cdrtools with mkisofs + ftp://ftp.berlios.de/pub/cdrecord/alpha + +ECMA-119 aka ISO 9660 + http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-119.pdf + +SUSP 1.12 + ftp://ftp.ymi.com/pub/rockridge/susp112.ps + +RRIP 1.12 + ftp://ftp.ymi.com/pub/rockridge/rrip112.ps + +--------------------------------------------------------------------------- + +This text is under +Copyright (c) 2009 - 2010, 2020 Thomas SCHMITT +Copyright (c) 2020 - Valentín KIVACHUK BURDÁ +It shall reflect the effective technical specifications as implemented in +zisofs2-tools and the Linux kernel. So please contact mailing list + or to the copyright holders in private, if you +want to make changes. +Only if you cannot reach the copyright holder for at least one month it is +permissible to modify and distribute this text under the license "GPLv3". + diff --git a/libisofs/filters/zisofs.c b/libisofs/filters/zisofs.c index 1398d50..5c45490 100644 --- a/libisofs/filters/zisofs.c +++ b/libisofs/filters/zisofs.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009 - 2011 Thomas Schmitt + * Copyright (c) 2009 - 2020 Thomas Schmitt * * This file is part of the libisofs project; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 @@ -23,6 +23,7 @@ #include "../fsource.h" #include "../util.h" #include "../stream.h" +#include "../messages.h" #include #include @@ -44,6 +45,45 @@ */ +/* The lowest size of a file which shall not be represented by zisofs v1 */ +#define ISO_ZISOFS_V1_LIMIT 4294967296 + +/* zisofs2: Test value for small mixed-version ISOs: 1 million + ISO_ZISOFS_V1_LIMIT 1000000 +*/ + +/* Limit for overall count of allocated block pointers: + 2 exp 25 = 256 MiB blocklist buffer = 4 TiB uncompressed at 128 KiB +*/ +#define ISO_ZISOFS_MAX_BLOCKS_T 0x2000000 + +/* Function to account for global block pointers */ +static uint64_t ziso_block_pointer_mgt(uint64_t num, int mode); + +/* Limit for single files: + 2 exp 25 = 256 MiB blocklist buffer = 4 TiB uncompressed at 128 KiB +*/ +#define ISO_ZISOFS_MAX_BLOCKS_F 0x2000000 + +#ifdef Not_yeT + +/* The number of blocks from which on the block pointer list shall be discarded + * on iso_stream_close() of a compressing stream. This means that list and size + * have to be determined again on next ziso_stream_get_size(). + * zisofs v1 with uint32_t pointers could at most have 131072 pointers. + * Since pointers are now uint64_t, the limit tolerates half of this. + */ +#define ISO_ZISOFS_MANY_BLOCKS 65537 + +#endif /* Not_yeT */ + +/* Minimum and maximum blocks sizes for version 1 and 2 */ +#define ISO_ZISOFS_V1_MIN_LOG2 15 +#define ISO_ZISOFS_V1_MAX_LOG2 17 +#define ISO_ZISOFS_V2_MIN_LOG2 15 +#define ISO_ZISOFS_V2_MAX_LOG2 20 + + /* --------------------------- ZisofsFilterRuntime ------------------------- */ @@ -51,6 +91,13 @@ static uint8_t ziso_block_size_log2 = 15; static int ziso_block_size = 32768; +static int ziso_v2_enabled = 0; +static int ziso_v2_block_size_log2 = 17; +static int ziso_v2_block_size = 1 << 17; + +static int64_t ziso_max_total_blocks = ISO_ZISOFS_MAX_BLOCKS_T; +static int64_t ziso_max_file_blocks = ISO_ZISOFS_MAX_BLOCKS_F; + /* Individual runtime properties exist only as long as the stream is opened. */ @@ -58,10 +105,12 @@ typedef struct { int state; /* processing: 0= header, 1= block pointers, 2= data blocks */ + int zisofs_version; /* 1 or 2 */ + int block_size; - int block_pointer_fill; - int block_pointer_rpos; - uint32_t *block_pointers; /* These are in use only with uncompression. + int64_t block_pointer_fill; + int64_t block_pointer_rpos; + uint64_t *block_pointers; /* These are in use only with uncompression. Compression streams hold the pointer in their persistent data. */ @@ -87,8 +136,10 @@ int ziso_running_destroy(ZisofsFilterRuntime **running, int flag) ZisofsFilterRuntime *o= *running; if (o == NULL) return 0; - if (o->block_pointers != NULL) + if (o->block_pointers != NULL) { + ziso_block_pointer_mgt((uint64_t) o->block_pointer_fill, 2); free(o->block_pointers); + } if (o->read_buffer != NULL) free(o->read_buffer); if (o->block_buffer != NULL) @@ -101,6 +152,7 @@ int ziso_running_destroy(ZisofsFilterRuntime **running, int flag) /* * @param flag bit0= do not set block_size, do not allocate buffers + * bit1= use ziso_v2_block_size */ static int ziso_running_new(ZisofsFilterRuntime **running, int flag) @@ -112,6 +164,7 @@ int ziso_running_new(ZisofsFilterRuntime **running, int flag) } o->state = 0; o->block_size= 0; + o->zisofs_version = 0; o->block_pointer_fill = 0; o->block_pointer_rpos = 0; o->block_pointers = NULL; @@ -128,11 +181,14 @@ int ziso_running_new(ZisofsFilterRuntime **running, int flag) if (flag & 1) return 1; - o->block_size = ziso_block_size; + if (flag & 2) + o->block_size = ziso_v2_block_size; + else + o->block_size = ziso_block_size; #ifdef Libisofs_with_zliB - o->buffer_size = compressBound((uLong) ziso_block_size); + o->buffer_size = compressBound((uLong) o->block_size); #else - o->buffer_size = 2 * ziso_block_size; + o->buffer_size = 2 * o->block_size; #endif o->read_buffer = calloc(o->block_size, 1); o->block_buffer = calloc(o->buffer_size, 1); @@ -145,12 +201,52 @@ failed: } +/* --------------------------- Resource accounting ------------------------- */ + +/* @param mode 0= inquire whether num block pointers would fit + 1= register num block pointers + 2= unregister num block_pointers + 3= return number of accounted block pointers + @return if not mode 3: 0= does not fit , 1= fits +*/ +static +uint64_t ziso_block_pointer_mgt(uint64_t num, int mode) +{ + static uint64_t global_count = 0; + static int underrun = 0; + + if (mode == 2) { + if (global_count < num) { + if (underrun < 3) + iso_msg_submit(-1, ISO_ZISOFS_BPT_UNDERRUN, 0, + "Prevented global block pointer counter underrun"); + underrun++; + global_count = 0; + } else { + global_count -= num; + } + } else if (mode == 3) { + return global_count; + } else { + if (global_count + num > (uint64_t) ziso_max_total_blocks) + return 0; + if (mode == 1) + global_count += num; + } + return 1; +} + + /* ---------------------------- ZisofsFilterStreamData --------------------- */ /* The first 8 bytes of a zisofs compressed data file */ static unsigned char zisofs_magic[9] = {0x37, 0xE4, 0x53, 0x96, 0xC9, 0xDB, 0xD6, 0x07}; +/* The first 8 bytes of a zisofs2 compressed data file */ +static unsigned char zisofs2_magic[9] = + {0xEF, 0x22, 0x55, 0xA1, 0xBC, 0x1B, 0x95, 0xA0}; + /* Counts the number of active compression filters */ static off_t ziso_ref_count = 0; @@ -191,11 +287,13 @@ typedef struct { ZisofsFilterStreamData std; - uint32_t orig_size; - uint32_t *block_pointers; /* Cache for output block addresses. They get + uint64_t orig_size; + uint64_t *block_pointers; /* Cache for output block addresses. They get written before the data and so need 2 passes. This cache avoids surplus passes. */ + uint64_t block_pointer_counter; + uint64_t open_counter; } ZisofsComprStreamData; @@ -207,6 +305,7 @@ typedef struct { ZisofsFilterStreamData std; + uint8_t zisofs_algo_num; unsigned char header_size_div4; unsigned char block_size_log2; @@ -227,18 +326,61 @@ static ino_t ziso_ino_id = 0; static int ziso_stream_uncompress(IsoStream *stream, void *buf, size_t desired); +static +int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired); + + +static +int ziso_decide_v2_usage(off_t orig_size) +{ + if (ziso_v2_enabled > 1 || + (ziso_v2_enabled == 1 && orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT)) + return 1; + return 0; +} + + /* * @param flag bit0= original stream is not open + * bit1= do not destroy large + * ZisofsComprStreamData->block_pointers */ static int ziso_stream_close_flag(IsoStream *stream, int flag) { ZisofsFilterStreamData *data; + ZisofsComprStreamData *cstd = NULL; if (stream == NULL) { return ISO_NULL_POINTER; } data = stream->data; + if (stream->class->read == &ziso_stream_compress) + cstd = (ZisofsComprStreamData *) data; + +#ifdef Not_yeT + + /* >>> zisofs2: + research whether zisofs streams get opened and closed often */ + + if (cstd != NULL) { + int block_size; + + block_size = ziso_block_size; + if (ziso_decide_v2_usage(cstd->orig_size)) + block_size = ziso_v2_block_size; + if ((!(flag & 2)) && cstd->open_counter == 1 && + cstd->orig_size / block_size >= ISO_ZISOFS_MANY_BLOCKS) { + if (cstd->block_pointers != NULL) { + ziso_block_pointer_mgt(cstd->block_pointer_counter, 2); + free((char *) cstd->block_pointers); + } + cstd->block_pointers = NULL; + data->size = -1; + } + } + +#endif /* Not_yeT */ if (data->running == NULL) { return 1; @@ -246,6 +388,9 @@ int ziso_stream_close_flag(IsoStream *stream, int flag) ziso_running_destroy(&(data->running), 0); if (flag & 1) return 1; + if (cstd != NULL) + if (cstd->open_counter > 0) + cstd->open_counter--; return iso_stream_close(data->orig); } @@ -264,8 +409,9 @@ static int ziso_stream_open_flag(IsoStream *stream, int flag) { ZisofsFilterStreamData *data; + ZisofsComprStreamData *cstd; ZisofsFilterRuntime *running = NULL; - int ret; + int ret, use_v2 = 0; if (stream == NULL) { return ISO_NULL_POINTER; @@ -280,9 +426,15 @@ int ziso_stream_open_flag(IsoStream *stream, int flag) */ stream->class->get_size(stream); } + if (stream->class->read == &ziso_stream_compress) { + cstd = (ZisofsComprStreamData *) data; + cstd->open_counter++; + use_v2 = ziso_decide_v2_usage((off_t) cstd->orig_size); + } ret = ziso_running_new(&running, - stream->class->read == &ziso_stream_uncompress); + (stream->class->read == &ziso_stream_uncompress) | + ((!!use_v2) << 1)); if (ret < 0) { return ret; } @@ -316,6 +468,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) off_t orig_size, next_pt; char *cbuf = buf; uLongf buf_len; + uint64_t *copy_base, num_blocks = 0; if (stream == NULL) { return ISO_NULL_POINTER; @@ -334,18 +487,39 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) /* Delivering file header */ if (rng->buffer_fill == 0) { - memcpy(rng->block_buffer, zisofs_magic, 8); orig_size = iso_stream_get_size(data->std.orig); - if (orig_size > 4294967295.0) { + num_blocks = orig_size / rng->block_size + + 1 + !!(orig_size % rng->block_size); + if (num_blocks > (uint64_t) ziso_max_file_blocks) return (rng->error_ret = ISO_ZISOFS_TOO_LARGE); + if (ziso_block_pointer_mgt((uint64_t) num_blocks, 0) == 0) + return (rng->error_ret = ISO_ZISOFS_TOO_MANY_PTR); + if (orig_size != (off_t) data->orig_size) + return (rng->error_ret = ISO_FILTER_WRONG_INPUT); + if (ziso_decide_v2_usage(orig_size)) { + rng->zisofs_version = 2; + memcpy(rng->block_buffer, zisofs2_magic, 8); + rng->block_buffer[8] = 0; /* @hdr_version */ + rng->block_buffer[9] = 6; /* @hdr_size */ + rng->block_buffer[10] = 1; /* @alg_id */ + rng->block_buffer[11] = ziso_v2_block_size_log2; + iso_lsb64((uint8_t *) (rng->block_buffer + 12), + (uint64_t) orig_size); + memset(rng->block_buffer + 20, 0, 4); + rng->buffer_fill = 24; + } else { + if (orig_size >= (off_t) ISO_ZISOFS_V1_LIMIT) { + return (rng->error_ret = ISO_ZISOFS_TOO_LARGE); + } + rng->zisofs_version = 1; + memcpy(rng->block_buffer, zisofs_magic, 8); + iso_lsb((unsigned char *) (rng->block_buffer + 8), + (uint32_t) orig_size, 4); + rng->block_buffer[12] = 4; + rng->block_buffer[13] = ziso_block_size_log2; + rng->block_buffer[14] = rng->block_buffer[15] = 0; + rng->buffer_fill = 16; } - data->orig_size = orig_size; - iso_lsb((unsigned char *) (rng->block_buffer + 8), - (uint32_t) orig_size, 4); - rng->block_buffer[12] = 4; - rng->block_buffer[13] = ziso_block_size_log2; - rng->block_buffer[14] = rng->block_buffer[15] = 0; - rng->buffer_fill = 16; rng->buffer_rpos = 0; } else if (rng->buffer_rpos >= rng->buffer_fill) { rng->buffer_fill = rng->buffer_rpos = 0; @@ -358,17 +532,25 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) if (rng->block_pointer_fill == 0) { /* Initialize block pointer writing */ rng->block_pointer_rpos = 0; - rng->block_pointer_fill = data->orig_size / rng->block_size - + 1 + !!(data->orig_size % rng->block_size); + num_blocks = data->orig_size / rng->block_size + + 1 + !!(data->orig_size % rng->block_size); + rng->block_pointer_fill = num_blocks; if (data->block_pointers == NULL) { /* On the first pass, create pointer array with all 0s */ - data->block_pointers = calloc(rng->block_pointer_fill, 4); + if (ziso_block_pointer_mgt(num_blocks, 1) == 0) { + rng->block_pointer_fill = 0; + return (rng->error_ret = ISO_ZISOFS_TOO_MANY_PTR); + } + data->block_pointers = calloc(rng->block_pointer_fill, 8); if (data->block_pointers == NULL) { + ziso_block_pointer_mgt(num_blocks, 2); rng->block_pointer_fill = 0; return (rng->error_ret = ISO_OUT_OF_MEM); } + data->block_pointer_counter = rng->block_pointer_fill; } } + if (rng->buffer_rpos >= rng->buffer_fill) { if (rng->block_pointer_rpos >= rng->block_pointer_fill) { rng->buffer_fill = rng->buffer_rpos = 0; @@ -377,14 +559,26 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) rng->state = 2; /* block pointers are delivered */ } else { /* Provide a buffer full of block pointers */ + /* data->block_pointers was filled by ziso_stream_open() */ todo = rng->block_pointer_fill - rng->block_pointer_rpos; - if (todo * 4 > rng->buffer_size) - todo = rng->buffer_size / 4; - memcpy(rng->block_buffer, - data->block_pointers + rng->block_pointer_rpos, - todo * 4); + copy_base = data->block_pointers + rng->block_pointer_rpos; + if (rng->zisofs_version == 1) { + if (todo * 4 > rng->buffer_size) + todo = rng->buffer_size / 4; + for (i = 0; i < todo; i++) + iso_lsb((unsigned char *) (rng->block_buffer + + 4 * i), + (uint32_t) (copy_base[i] & 0xffffffff), 4); + rng->buffer_fill = todo * 4; + } else { + if (todo * 8 > rng->buffer_size) + todo = rng->buffer_size / 8; + for (i = 0; i < todo; i++) + iso_lsb64((uint8_t *) rng->block_buffer + 8 * i, + copy_base[i]); + rng->buffer_fill = todo * 8; + } rng->buffer_rpos = 0; - rng->buffer_fill = todo * 4; rng->block_pointer_rpos += todo; } } @@ -396,7 +590,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) rng->block_size); if (ret > 0) { rng->in_counter += ret; - if (rng->in_counter > data->orig_size) { + if ((uint64_t) rng->in_counter > data->orig_size) { /* Input size became larger */ return (rng->error_ret = ISO_FILTER_WRONG_INPUT); } @@ -425,10 +619,11 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) return (rng->error_ret = ISO_FILTER_WRONG_INPUT); } - /* Record resp. check block pointer */ + /* Check or record check block pointer */ rng->block_counter++; if (data->block_pointers[rng->block_counter] > 0) { - if (next_pt != data->block_pointers[rng->block_counter] ) { + if ((uint64_t) next_pt != + data->block_pointers[rng->block_counter]) { /* block pointers mismatch , content has changed */ return (rng->error_ret = ISO_FILTER_WRONG_INPUT); } @@ -438,7 +633,7 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) } else if (ret == 0) { rng->state = 3; - if (rng->in_counter != data->orig_size) { + if ((uint64_t) rng->in_counter != data->orig_size) { /* Input size shrunk */ return (rng->error_ret = ISO_FILTER_WRONG_INPUT); } @@ -478,24 +673,91 @@ int ziso_stream_compress(IsoStream *stream, void *buf, size_t desired) static -int ziso_parse_zisofs_head(IsoStream *stream, int *header_size_div4, - int *block_size_log2, uint32_t *uncompressed_size, - int flag) +int ziso_algo_to_num(uint8_t zisofs_algo[2]) { - int ret; - char zisofs_head[16]; + if (zisofs_algo[0] == 'p' && zisofs_algo[1] == 'z') + return 0; + if (zisofs_algo[0] == 'P' && zisofs_algo[1] == 'Z') + return 1; + return -1; +} - ret = iso_stream_read(stream, zisofs_head, 16); + +static +int ziso_num_to_algo(uint8_t num, uint8_t zisofs_algo[2]) +{ + if (num == 0) { + zisofs_algo[0] = 'p'; + zisofs_algo[1] = 'z'; + return 1; + } else if (num == 1) { + zisofs_algo[0] = 'P'; + zisofs_algo[1] = 'Z'; + return 2; + } + return -1; +} + + +/* @param flag bit0= recognize zisofs2 only if ziso_v2_enabled +*/ +static +int ziso_parse_zisofs_head(IsoStream *stream, uint8_t *ziso_algo_num, + int *header_size_div4, int *block_size_log2, + uint64_t *uncompressed_size, int flag) +{ + int ret, consumed = 0, i; + char zisofs_head[24]; + char waste_word[4]; + + ret = iso_stream_read(stream, zisofs_head, 8); if (ret < 0) return ret; - *header_size_div4 = ((unsigned char *) zisofs_head)[12]; - *block_size_log2 = ((unsigned char *) zisofs_head)[13]; - if (ret != 16 || memcmp(zisofs_head, zisofs_magic, 8) != 0 || - *header_size_div4 < 4 || - *block_size_log2 < 15 || *block_size_log2 > 17) { + if (ret != 8) + return ISO_ZISOFS_WRONG_INPUT; + consumed = 8; + if (memcmp(zisofs_head, zisofs_magic, 8) == 0) { + *ziso_algo_num = 0; + ret = iso_stream_read(stream, zisofs_head + 8, 8); + if (ret < 0) + return ret; + if (ret != 8) + return ISO_ZISOFS_WRONG_INPUT; + consumed += 8; + *header_size_div4 = ((unsigned char *) zisofs_head)[12]; + *block_size_log2 = ((unsigned char *) zisofs_head)[13]; + *uncompressed_size = iso_read_lsb(((uint8_t *) zisofs_head) + 8, 4); + if (*header_size_div4 < 4 || + *block_size_log2 < ISO_ZISOFS_V1_MIN_LOG2 || + *block_size_log2 > ISO_ZISOFS_V1_MAX_LOG2) + return ISO_ZISOFS_WRONG_INPUT; + } else if (memcmp(zisofs_head, zisofs2_magic, 8) == 0 && + !(ziso_v2_enabled == 0 && (flag & 1))) { + ret = iso_stream_read(stream, zisofs_head + 8, 16); + if (ret < 0) + return ret; + if (ret != 16) + return ISO_ZISOFS_WRONG_INPUT; + consumed += 16; + *ziso_algo_num = zisofs_head[10]; + *header_size_div4 = ((unsigned char *) zisofs_head)[9]; + *block_size_log2 = ((unsigned char *) zisofs_head)[11]; + *uncompressed_size = iso_read_lsb64(((uint8_t *) zisofs_head) + 12); + if (*header_size_div4 < 4 || + *block_size_log2 < ISO_ZISOFS_V2_MIN_LOG2 || + *block_size_log2 > ISO_ZISOFS_V2_MAX_LOG2 || *ziso_algo_num != 1) + return ISO_ZISOFS_WRONG_INPUT; + } else { return ISO_ZISOFS_WRONG_INPUT; } - *uncompressed_size = iso_read_lsb(((uint8_t *) zisofs_head) + 8, 4); + for (i = consumed; i < *header_size_div4; i++) { + /* Skip surplus header words */ + ret = iso_stream_read(stream, waste_word, 4); + if (ret < 0) + return ret; + if (ret != 4) + return ISO_ZISOFS_WRONG_INPUT; + } return 1; } @@ -509,15 +771,16 @@ int ziso_stream_uncompress(IsoStream *stream, void *buf, size_t desired) #ifdef Libisofs_with_zliB - int ret, todo, i, header_size, bs_log2, block_max = 1; + int ret, todo, header_size, bs_log2, block_max = 1, blpt_size; ZisofsFilterStreamData *data; ZisofsFilterRuntime *rng; ZisofsUncomprStreamData *nstd; size_t fill = 0; char *cbuf = buf; uLongf buf_len; - uint32_t uncompressed_size; - char waste_word[4]; + uint64_t uncompressed_size; + int64_t i; + uint8_t algo_num, *rpt, *wpt; if (stream == NULL) { return ISO_NULL_POINTER; @@ -535,46 +798,63 @@ int ziso_stream_uncompress(IsoStream *stream, void *buf, size_t desired) while (1) { if (rng->state == 0) { /* Reading file header */ - ret = ziso_parse_zisofs_head(data->orig, &header_size, &bs_log2, - &uncompressed_size, 0); + ret = ziso_parse_zisofs_head(data->orig, &algo_num, &header_size, + &bs_log2, &uncompressed_size, 0); if (ret < 0) return (rng->error_ret = ret); + if (algo_num == 0) + blpt_size = 4; + else + blpt_size = 8; nstd->header_size_div4 = header_size; header_size *= 4; data->size = uncompressed_size; nstd->block_size_log2 = bs_log2; rng->block_size = 1 << bs_log2; - for (i = 16; i < header_size; i += 4) { - /* Skip surplus header words */ - ret = iso_stream_read(data->orig, waste_word, 4); - if (ret < 0) - return (rng->error_ret = ret); - if (ret != 4) - return (rng->error_ret = ISO_ZISOFS_WRONG_INPUT); - } - if (desired == 0) { + if (desired == 0) return 0; - } /* Create and read pointer array */ rng->block_pointer_rpos = 0; rng->block_pointer_fill = data->size / rng->block_size + 1 + !!(data->size % rng->block_size); - rng->block_pointers = calloc(rng->block_pointer_fill, 4); + if (rng->block_pointer_fill > ziso_max_file_blocks) { + rng->block_pointer_fill = 0; + return (rng->error_ret = ISO_ZISOFS_TOO_LARGE); + } + if (ziso_block_pointer_mgt((uint64_t) rng->block_pointer_fill, 1) + == 0) + return ISO_ZISOFS_TOO_MANY_PTR; + rng->block_pointers = calloc(rng->block_pointer_fill, 8); if (rng->block_pointers == NULL) { + ziso_block_pointer_mgt((uint64_t) rng->block_pointer_fill, 2); rng->block_pointer_fill = 0; return (rng->error_ret = ISO_OUT_OF_MEM); } ret = iso_stream_read(data->orig, rng->block_pointers, - rng->block_pointer_fill * 4); + rng->block_pointer_fill * blpt_size); if (ret < 0) return (rng->error_ret = ret); - if (ret != rng->block_pointer_fill * 4) + if (algo_num == 0) { + /* Spread 4 byte little-endian pointer values over 8 byte */ + rpt = ((uint8_t *) rng->block_pointers) + + rng->block_pointer_fill * 4; + wpt = ((uint8_t *) rng->block_pointers) + + rng->block_pointer_fill * 8; + while (rpt > ((uint8_t *) rng->block_pointers) + 4) { + rpt -= 4; + wpt -= 8; + memcpy(wpt, rpt, 4); + memset(wpt + 4, 0, 4); + } + memset(((uint8_t *) rng->block_pointers) + 4, 0, 4); + } + if (ret != rng->block_pointer_fill * blpt_size) return (rng->error_ret = ISO_ZISOFS_WRONG_INPUT); for (i = 0; i < rng->block_pointer_fill; i++) { rng->block_pointers[i] = - iso_read_lsb((uint8_t *) (rng->block_pointers + i), 4); + iso_read_lsb64((uint8_t *) (rng->block_pointers + i)); if (i > 0) if ((int) (rng->block_pointers[i] - rng->block_pointers[i - 1]) @@ -704,7 +984,7 @@ off_t ziso_stream_get_size(IsoStream *stream) count += ret; } } - ret_close = ziso_stream_close(stream); + ret_close = ziso_stream_close_flag(stream, 2); if (ret < 0) return ret; if (ret_close < 0) @@ -754,8 +1034,10 @@ void ziso_stream_free(IsoStream *stream) ziso_osiz_ref_count = 0; } else { nstd = stream->data; - if (nstd->block_pointers != NULL) + if (nstd->block_pointers != NULL) { + ziso_block_pointer_mgt(nstd->block_pointer_counter, 2); free((char *) nstd->block_pointers); + } if (--ziso_ref_count < 0) ziso_ref_count = 0; } @@ -807,6 +1089,7 @@ int ziso_clone_stream(IsoStream *old_stream, IsoStream **new_stream, int flag) goto no_mem; stream_data = (ZisofsFilterStreamData *) uncompr; old_uncompr = (ZisofsUncomprStreamData *) old_stream->data; + uncompr->zisofs_algo_num = old_uncompr->zisofs_algo_num; uncompr->header_size_div4 = old_uncompr->header_size_div4; uncompr->block_size_log2 = old_uncompr->block_size_log2; } else { @@ -817,6 +1100,8 @@ int ziso_clone_stream(IsoStream *old_stream, IsoStream **new_stream, int flag) old_compr = (ZisofsComprStreamData *) old_stream->data; compr->orig_size = old_compr->orig_size; compr->block_pointers = NULL; + compr->block_pointer_counter = 0; + compr->open_counter = 0; } old_stream_data = (ZisofsFilterStreamData *) old_stream->data; stream_data->orig = new_input_stream; @@ -971,13 +1256,16 @@ int ziso_filter_get_filter(FilterContext *filter, IsoStream *original, str->refcount = 1; str->data = data; if (flag & 2) { + unstd->zisofs_algo_num = 0; unstd->header_size_div4 = 0; unstd->block_size_log2 = 0; str->class = &ziso_stream_uncompress_class; ziso_osiz_ref_count++; } else { - cnstd->orig_size = 0; + cnstd->orig_size = iso_stream_get_size(original); cnstd->block_pointers = NULL; + cnstd->block_pointer_counter = 0; + cnstd->open_counter = 0; str->class = &ziso_stream_compress_class; ziso_ref_count++; } @@ -1059,7 +1347,7 @@ int ziso_add_filter(IsoFile *file, int flag) if (original_size <= 0 || ((flag & 1) && original_size <= 2048)) { return 2; } - if (original_size > 4294967295.0) { + if (original_size >= (off_t) ISO_ZISOFS_V1_LIMIT && !ziso_v2_enabled) { return ISO_ZISOFS_TOO_LARGE; } } @@ -1119,9 +1407,9 @@ int iso_zisofs_get_refcounts(off_t *ziso_count, off_t *osiz_count, int flag) } -int ziso_add_osiz_filter(IsoFile *file, uint8_t header_size_div4, - uint8_t block_size_log2, uint32_t uncompressed_size, - int flag) +int ziso_add_osiz_filter(IsoFile *file, uint8_t zisofs_algo[2], + uint8_t header_size_div4, uint8_t block_size_log2, + uint64_t uncompressed_size, int flag) { #ifdef Libisofs_with_zliB @@ -1133,6 +1421,10 @@ int ziso_add_osiz_filter(IsoFile *file, uint8_t header_size_div4, if (ret < 0) return ret; unstd = iso_file_get_stream(file)->data; + ret = ziso_algo_to_num(zisofs_algo); + if (ret < 0) + return ISO_ZISOFS_WRONG_INPUT; + unstd->zisofs_algo_num = ret; unstd->header_size_div4 = header_size_div4; unstd->block_size_log2 = block_size_log2; unstd->std.size = uncompressed_size; @@ -1152,28 +1444,44 @@ int ziso_add_osiz_filter(IsoFile *file, uint8_t header_size_div4, and eventual ZF field parameters @param flag bit0= allow ziso_by_content which is based on content reading bit1= do not inquire stream->class for filters + bit2= recognize zisofs2 by magic only if ziso_v2_enabled */ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type, + uint8_t zisofs_algo[2], int *header_size_div4, int *block_size_log2, - uint32_t *uncompressed_size, int flag) + uint64_t *uncompressed_size, int flag) { - int ret, close_ret; + int ret, close_ret, algo_ret; ZisofsFilterStreamData *data; ZisofsComprStreamData *cnstd; ZisofsUncomprStreamData *unstd; + uint8_t algo_num; *stream_type = 0; if (stream->class == &ziso_stream_compress_class && !(flag & 2)) { *stream_type = 1; cnstd = stream->data; *header_size_div4 = 4; - *block_size_log2 = ziso_block_size_log2; *uncompressed_size = cnstd->orig_size; + if (ziso_decide_v2_usage((off_t) *uncompressed_size)) { + *block_size_log2 = ziso_v2_block_size_log2; + zisofs_algo[0] = 'P'; + zisofs_algo[1] = 'Z'; + } else if (*uncompressed_size < (uint64_t) ISO_ZISOFS_V1_LIMIT) { + *block_size_log2 = ziso_block_size_log2; + zisofs_algo[0] = 'p'; + zisofs_algo[1] = 'z'; + } else { + return 0; + } return 1; } else if(stream->class == &ziso_stream_uncompress_class && !(flag & 2)) { *stream_type = -1; data = stream->data; unstd = stream->data; + ret = ziso_num_to_algo(unstd->zisofs_algo_num, zisofs_algo); + if (ret < 0) + return ISO_ZISOFS_WRONG_INPUT; *header_size_div4 = unstd->header_size_div4; *block_size_log2 = unstd->block_size_log2; *uncompressed_size = data->size; @@ -1185,14 +1493,19 @@ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type, ret = iso_stream_open(stream); if (ret < 0) return ret; - ret = ziso_parse_zisofs_head(stream, header_size_div4, - block_size_log2, uncompressed_size, 0); + ret = ziso_parse_zisofs_head(stream, &algo_num, header_size_div4, + block_size_log2, uncompressed_size, + (flag >> 2) & 1); if (ret == 1) { *stream_type = 2; + algo_ret = ziso_num_to_algo(algo_num, zisofs_algo); } else { ret = 0; + algo_ret = 1; } close_ret = iso_stream_close(stream); + if (algo_ret < 0) + return ISO_ZISOFS_WRONG_INPUT; if (close_ret < 0) return close_ret; @@ -1200,21 +1513,47 @@ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type, } +/* API */ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag) { #ifdef Libisofs_with_zliB + if (params->version < 0 || params->version > 1) + return ISO_WRONG_ARG_VALUE; + if (params->compression_level < 0 || params->compression_level > 9 || - params->block_size_log2 < 15 || params->block_size_log2 > 17) { + params->block_size_log2 < ISO_ZISOFS_V1_MIN_LOG2 || + params->block_size_log2 > ISO_ZISOFS_V1_MAX_LOG2) { return ISO_WRONG_ARG_VALUE; } + if (params->version >= 1) + if (params->v2_enabled < 0 || params->v2_enabled > 2 || + (params->v2_block_size_log2 != 0 && + (params->v2_block_size_log2 < ISO_ZISOFS_V2_MIN_LOG2 || + params->v2_block_size_log2 > ISO_ZISOFS_V2_MAX_LOG2))) + return ISO_WRONG_ARG_VALUE; if (ziso_ref_count > 0) { return ISO_ZISOFS_PARAM_LOCK; } ziso_compression_level = params->compression_level; ziso_block_size_log2 = params->block_size_log2; ziso_block_size = 1 << ziso_block_size_log2; + + if (params->version == 0) + return 1; + + ziso_v2_enabled = params->v2_enabled; + if (params->v2_block_size_log2 > 0) + ziso_v2_block_size_log2 = params->v2_block_size_log2; + ziso_v2_block_size = 1 << ziso_v2_block_size_log2; + if (params->max_total_blocks > 0) + ziso_max_total_blocks = params->max_total_blocks; + if (params->max_file_blocks > 0) + ziso_max_file_blocks = params->max_file_blocks; + + /* >>> zisofs2: more parameters */ + return 1; #else @@ -1226,13 +1565,24 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag) } +/* API */ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag) { #ifdef Libisofs_with_zliB + if (params->version < 0 || params->version > 1) + return ISO_WRONG_ARG_VALUE; + params->compression_level = ziso_compression_level; params->block_size_log2 = ziso_block_size_log2; + if (params->version == 1) { + params->v2_enabled = ziso_v2_enabled; + params->v2_block_size_log2 = ziso_v2_block_size_log2; + params->max_total_blocks = ziso_max_total_blocks; + params->current_total_blocks = ziso_block_pointer_mgt((uint64_t) 0, 3); + params->max_file_blocks = ziso_max_file_blocks; + } return 1; #else @@ -1243,3 +1593,21 @@ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag) } + +/* API */ +int iso_stream_get_zisofs_par(IsoStream *stream, int *stream_type, + uint8_t zisofs_algo[2], uint8_t* algo_num, + int *block_size_log2, int flag) +{ + uint64_t uncompressed_size; + int header_size_div4, ret; + + ret = ziso_is_zisofs_stream(stream, stream_type, zisofs_algo, + &header_size_div4, block_size_log2, + &uncompressed_size, 0); + if (ret <= 0 || (*stream_type != -1 && *stream_type != 1)) + return 0; + *algo_num = ziso_algo_to_num(zisofs_algo); + return 1; +} + diff --git a/libisofs/fs_image.c b/libisofs/fs_image.c index 7522be6..ad0c640 100644 --- a/libisofs/fs_image.c +++ b/libisofs/fs_image.c @@ -418,9 +418,10 @@ struct image_fs_data unsigned int opened : 2; /**< 0 not opened, 1 opened file, 2 opened dir */ #ifdef Libisofs_with_zliB + uint8_t zisofs_algo[2]; uint8_t header_size_div4; uint8_t block_size_log2; - uint32_t uncompressed_size; + uint64_t uncompressed_size; #endif /* info for content reading */ @@ -1252,9 +1253,9 @@ IsoFileSourceIface ifs_class = { /* Used from libisofs/stream.c : iso_stream_get_src_zf() */ -int iso_ifs_source_get_zf(IsoFileSource *src, int *header_size_div4, - int *block_size_log2, uint32_t *uncompressed_size, - int flag) +int iso_ifs_source_get_zf(IsoFileSource *src, uint8_t zisofs_algo[2], + int *header_size_div4, int *block_size_log2, + uint64_t *uncompressed_size, int flag) { #ifdef Libisofs_with_zliB @@ -1264,6 +1265,8 @@ int iso_ifs_source_get_zf(IsoFileSource *src, int *header_size_div4, if (src->class != &ifs_class) return 0; data = src->data; + zisofs_algo[0] = data->zisofs_algo[0]; + zisofs_algo[1] = data->zisofs_algo[1]; *header_size_div4 = data->header_size_div4; *block_size_log2 = data->block_size_log2; *uncompressed_size = data->uncompressed_size; @@ -1460,7 +1463,7 @@ int iso_file_source_new_ifs(IsoImageFilesystem *fs, IsoFileSource *parent, #ifdef Libisofs_with_zliB uint8_t zisofs_alg[2], zisofs_hs4 = 0, zisofs_bsl2 = 0; - uint32_t zisofs_usize = 0; + uint64_t zisofs_usize = 0; #endif if (fs == NULL || fs->data == NULL || record == NULL || src == NULL) { @@ -1593,7 +1596,7 @@ int iso_file_source_new_ifs(IsoImageFilesystem *fs, IsoFileSource *parent, while ((ret = susp_iter_next(iter, &sue, 0)) > 0) { /* ignore entries from different version */ - if (sue->version[0] != 1) + if (sue->version[0] != 1 && !(SUSP_SIG(sue, 'Z', 'F'))) continue; if (SUSP_SIG(sue, 'P', 'X')) { @@ -1804,13 +1807,23 @@ if (name != NULL && !namecont) { ret = read_zisofs_ZF(sue, zisofs_alg, &zisofs_hs4, &zisofs_bsl2, &zisofs_usize, 0); - if (ret < 0 || zisofs_alg[0] != 'p' || zisofs_alg[1] != 'z') { + if (ret < 0) { +invalid_zf: /* notify and continue */ ret = iso_rr_msg_submit(fsdata, 13, ISO_WRONG_RR_WARN, ret, "Invalid ZF entry"); zisofs_hs4 = 0; continue; } + if (zisofs_alg[0] == 'p' || zisofs_alg[1] == 'z') { + if (sue->version[0] != 1) + goto invalid_zf; + } else if (zisofs_alg[0] == 'P' || zisofs_alg[1] == 'Z') { + if (sue->version[0] != 2) + goto invalid_zf; + } else { + goto invalid_zf; + } #endif /* Libisofs_with_zliB */ @@ -2085,6 +2098,8 @@ if (name != NULL && !namecont) { #ifdef Libisofs_with_zliB if (zisofs_hs4 > 0) { + ifsdata->zisofs_algo[0] = zisofs_alg[0]; + ifsdata->zisofs_algo[1] = zisofs_alg[1]; ifsdata->header_size_div4 = zisofs_hs4; ifsdata->block_size_log2 = zisofs_bsl2; ifsdata->uncompressed_size = zisofs_usize; @@ -3172,8 +3187,8 @@ int image_builder_create_node(IsoNodeBuilder *builder, IsoImage *image, #ifdef Libisofs_with_zliB /* Intimate friendship with this function in filters/zisofs.c */ - int ziso_add_osiz_filter(IsoFile *file, uint8_t header_size_div4, - uint8_t block_size_log2, + int ziso_add_osiz_filter(IsoFile *file, uint8_t zisofs_algo[2], + uint8_t header_size_div4, uint8_t block_size_log2, uint32_t uncompressed_size, int flag); #endif /* Libisofs_with_zliB */ @@ -3288,7 +3303,8 @@ int image_builder_create_node(IsoNodeBuilder *builder, IsoImage *image, #ifdef Libisofs_with_zliB if (data->header_size_div4 > 0) { - ret = ziso_add_osiz_filter(file, data->header_size_div4, + ret = ziso_add_osiz_filter(file, data->zisofs_algo, + data->header_size_div4, data->block_size_log2, data->uncompressed_size, 0); if (ret < 0) { diff --git a/libisofs/libisofs.h b/libisofs/libisofs.h index a20f047..586384b 100644 --- a/libisofs/libisofs.h +++ b/libisofs/libisofs.h @@ -4,7 +4,7 @@ /* * Copyright (c) 2007-2008 Vreixo Formoso, Mario Danic - * Copyright (c) 2009-2019 Thomas Schmitt + * Copyright (c) 2009-2020 Thomas Schmitt * * This file is part of the libisofs project; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 @@ -7942,7 +7942,7 @@ int iso_stream_get_external_filter(IsoStream *stream, /** * Install a zisofs filter on top of the content stream of a data file. * zisofs is a compression format which is decompressed by some Linux kernels. - * See also doc/zisofs_format.txt . + * See also doc/zisofs_format.txt and doc/zisofs2_format.txt. * The filter will not be installed if its output size is not smaller than * the size of the input stream. * This is only enabled if the use of libz was enabled at compile time. @@ -7964,6 +7964,37 @@ int iso_stream_get_external_filter(IsoStream *stream, */ int iso_file_add_zisofs_filter(IsoFile *file, int flag); + +/** + * Obtain the parameters of a zisofs filter stream. + * @param stream + * The stream to be inquired. + * @param stream_type + * 1=compressing ("ziso") + * -1=uncompressing ("osiz") + * 0 other (any obtained parameters have invalid content) + * @param zisofs_algo + * Algorithm as of ZF field: + * {'p', 'z'} = zisofs version 1 (Zlib) + * {'P', 'Z'} = zisofs version 2 (Zlib) + * @param algo_num + * Algorithm as of zisofs header: + * 0 = zisofs version 1 (Zlib) + * 1 = zisofs version 2 (Zlib) + * @param block_size_log2 + * Log2 of the compression block size + * 15 = 32 kiB , 16 = 64 kiB , 17 = 128 kiB, ... + * @param flag + * Bitfield for control purposes, unused yet, submit 0 + * @return + * 1 on success, 0 if the stream has not class->type "ziso" or "osiz" + * @since 1.5.4 + */ +int iso_stream_get_zisofs_par(IsoStream *stream, int *stream_type, + uint8_t zisofs_algo[2], uint8_t* algo_num, + int *block_size_log2, int flag); + + /** * Inquire the number of zisofs compression and uncompression filters which * are in use. @@ -7988,7 +8019,11 @@ int iso_zisofs_get_refcounts(off_t *ziso_count, off_t *osiz_count, int flag); */ struct iso_zisofs_ctrl { - /* Set to 0 for this version of the structure */ + /* Set to 0 or 1 for this version of the structure + * 0 = only members up to .block_size_log2 are valid + * 1 = members up to .max_file_blocks are valid + * @since 1.5.4 + */ int version; /* Compression level for zlib function compress2(). From : @@ -7998,11 +8033,59 @@ struct iso_zisofs_ctrl { */ int compression_level; - /* Log2 of the block size for compression filters. Allowed values are: + /* Log2 of the block size for compression filters of zisofs version 1. + * Allowed values are: * 15 = 32 kiB , 16 = 64 kiB , 17 = 128 kiB */ uint8_t block_size_log2; + /* ------------------- Only valid with .version >= 1 ------------------- */ + + /* + * @since 1.5.4 + * Whether to produce zisofs2 (zisofs version 2) file headers and ZF + * entries for files which get compressed: + * 0 = do not produce zisofs2, + * do not recognize zisofs2 file headers by magic + * This is the default. + * 1 = zisofs2 is enabled for file size 4 GiB or more + * 2 = zisofs2 shall be used if zisofs is used at all + */ + int v2_enabled; + + /* + * @since 1.5.4 + * Log2 of block size for zisofs2 files. 0 keeps current setting. + * Allowed are 15 = 32 kiB to 20 = 1024 kiB. + */ + uint8_t v2_block_size_log2; + + /* + * @since 1.5.4 + * Maximum overall number of blocklist pointers. 0 keeps current setting. + */ + uint64_t max_total_blocks; + + /* + * @since 1.5.4 + * Ignored as input value: Number of allocated zisofs block pointers. + */ + uint64_t current_total_blocks; + + /* + * @since 1.5.4 + * Maximum number of blocklist pointers per file. 0 keeps current setting. + */ + uint64_t max_file_blocks; + + /* >>> ??? zisofs2: ISO_ZISOFS_MANY_BLOCKS , 0 = default 65537 */ + + /* >>> ??? zisofs2: a limit for number of zisofs2 files in order to keep + the number of these old kernel warnings bearable: + "isofs: Unknown ZF compression algorithm: PZ" + 0 = default >>> ??? value ? no limit ? + */ + }; /** @@ -8011,6 +8094,9 @@ struct iso_zisofs_ctrl { * i.e. ziso_count returned by iso_zisofs_get_refcounts() has to be 0. * @param params * Pointer to a structure with the intended settings. + * The caller sets params->version to indicate which set of members + * has been filled. I.e. params->version == 0 causes all members after + * params->block_size_log2 to be ignored. * @param flag * Bitfield for control purposes, unused yet, submit 0 * @return @@ -8024,6 +8110,9 @@ int iso_zisofs_set_params(struct iso_zisofs_ctrl *params, int flag); * Get the current global parameters for zisofs filtering. * @param params * Pointer to a caller provided structure which shall take the settings. + * The caller sets params->version to indicate which set of members + * shall be filled. I.e. params->version == 0 leaves all members after + * params->block_size_log2 untouched. * @param flag * Bitfield for control purposes, unused yet, submit 0 * @return @@ -8040,7 +8129,7 @@ int iso_zisofs_get_params(struct iso_zisofs_ctrl *params, int flag); * by an xinfo data record if not already marked by a zisofs compressor filter. * This does not install any filter but only a hint for image generation * that the already compressed files shall get written with zisofs ZF entries. - * Use this if you insert the compressed reults of program mkzftree from disk + * Use this if you insert the compressed results of program mkzftree from disk * into the image. * @param node * The node which shall be checked and eventually marked. @@ -8756,7 +8845,7 @@ int iso_conv_name_chars(IsoWriteOpts *opts, char *name, size_t name_len, /** Use of zlib was not enabled at compile time (FAILURE, HIGH, -345) */ #define ISO_ZLIB_NOT_ENABLED 0xE830FEA7 -/** Cannot apply zisofs filter to file >= 4 GiB (FAILURE, HIGH, -346) */ +/** File too large. Cannot apply zisofs filter. (FAILURE, HIGH, -346) */ #define ISO_ZISOFS_TOO_LARGE 0xE830FEA6 /** Filter input differs from previous run (FAILURE, HIGH, -347) */ @@ -8765,7 +8854,7 @@ int iso_conv_name_chars(IsoWriteOpts *opts, char *name, size_t name_len, /** zlib compression/decompression error (FAILURE, HIGH, -348) */ #define ISO_ZLIB_COMPR_ERR 0xE830FEA4 -/** Input stream is not in zisofs format (FAILURE, HIGH, -349) */ +/** Input stream is not in a supported zisofs format (FAILURE, HIGH, -349) */ #define ISO_ZISOFS_WRONG_INPUT 0xE830FEA3 /** Cannot set global zisofs parameters while filters exist @@ -9046,6 +9135,12 @@ int iso_conv_name_chars(IsoWriteOpts *opts, char *name, size_t name_len, /** Too many files in HFS+ directory tree (FAILURE, HIGH, -422) */ #define ISO_HFSPLUS_TOO_MANY_FILES 0xE830FE5A +/** Too many zisofs block pointers needed overall (FAILURE, HIGH, -423) */ +#define ISO_ZISOFS_TOO_MANY_PTR 0xE830FE59 + +/** Prevented zisofs block pointer counter underrun (WARNING,MEDIUM, -424) */ +#define ISO_ZISOFS_BPT_UNDERRUN 0xD020FE58 + /* Internal developer note: Place new error codes directly above this comment. diff --git a/libisofs/libisofs.ver b/libisofs/libisofs.ver index 72f9c6d..aca8bb5 100644 --- a/libisofs/libisofs.ver +++ b/libisofs/libisofs.ver @@ -266,6 +266,7 @@ iso_stream_get_id; iso_stream_get_input_stream; iso_stream_get_size; iso_stream_get_source_path; +iso_stream_get_zisofs_par; iso_stream_is_repeatable; iso_stream_open; iso_stream_read; diff --git a/libisofs/messages.c b/libisofs/messages.c index 45b341d..400d12b 100644 --- a/libisofs/messages.c +++ b/libisofs/messages.c @@ -402,13 +402,13 @@ const char *iso_error_to_msg(int errcode) case ISO_ZLIB_NOT_ENABLED: return "Use of zlib was not enabled at compile time"; case ISO_ZISOFS_TOO_LARGE: - return "Cannot apply zisofs filter to file >= 4 GiB"; + return "File too large. Cannot apply zisofs filter."; case ISO_FILTER_WRONG_INPUT: return "Filter input differs from previous run"; case ISO_ZLIB_COMPR_ERR: return "zlib compression/decompression error"; case ISO_ZISOFS_WRONG_INPUT: - return "Input stream is not in zisofs format"; + return "Input stream is not in a supported zisofs format"; case ISO_ZISOFS_PARAM_LOCK: return "Cannot set global zisofs parameters while filters exist"; case ISO_ZLIB_EARLY_EOF: @@ -557,6 +557,10 @@ const char *iso_error_to_msg(int errcode) return "El-Torito EFI image is hidden"; case ISO_HFSPLUS_TOO_MANY_FILES: return "Too many files in HFS+ directory tree"; + case ISO_ZISOFS_TOO_MANY_PTR: + return "Too many zisofs block pointers needed overall"; + case ISO_ZISOFS_BPT_UNDERRUN: + return "Prevented zisofs block pointer counter underrun"; default: return "Unknown error"; } diff --git a/libisofs/node.c b/libisofs/node.c index 5c71d5d..0858d22 100644 --- a/libisofs/node.c +++ b/libisofs/node.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2007 Vreixo Formoso - * Copyright (c) 2009 - 2019 Thomas Schmitt + * Copyright (c) 2009 - 2020 Thomas Schmitt * * This file is part of the libisofs project; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 @@ -2447,15 +2447,17 @@ int zisofs_zf_xinfo_cloner(void *old_data, void **new_data, int flag) int iso_file_zf_by_magic(IsoFile *file, int flag) { int ret, stream_type, header_size_div4, block_size_log2; - uint32_t uncompressed_size; + uint64_t uncompressed_size; IsoStream *stream, *input_stream; struct zisofs_zf_info *zf = NULL; void *xipt; + uint8_t algo[2]; /* Intimate friendship with this function in filters/zisofs.c */ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type, + uint8_t zisofs_algo[2], int *header_size_div4, int *block_size_log2, - uint32_t *uncompressed_size, int flag); + uint64_t *uncompressed_size, int flag); ret = iso_node_get_xinfo((IsoNode *) file, zisofs_zf_xinfo_func, &xipt); if (ret == 1) { @@ -2472,13 +2474,14 @@ int iso_file_zf_by_magic(IsoFile *file, int flag) break; stream = input_stream; } - ret = ziso_is_zisofs_stream(stream, &stream_type, &header_size_div4, - &block_size_log2, &uncompressed_size, 3); + ret = ziso_is_zisofs_stream(stream, &stream_type, algo, &header_size_div4, + &block_size_log2, &uncompressed_size, 7); if (ret < 0) return ret; if (ret != 1 || stream_type != 2) { if (flag & 4) return 0; + algo[0] = algo[1] = 0; header_size_div4 = 0; block_size_log2 = 0; uncompressed_size = 0; @@ -2486,6 +2489,8 @@ int iso_file_zf_by_magic(IsoFile *file, int flag) zf = calloc(1, sizeof(struct zisofs_zf_info)); if (zf == NULL) return ISO_OUT_OF_MEM; + zf->zisofs_algo[0] = algo[0]; + zf->zisofs_algo[1] = algo[1]; zf->uncompressed_size = uncompressed_size; zf->header_size_div4 = header_size_div4; zf->block_size_log2 = block_size_log2; diff --git a/libisofs/node.h b/libisofs/node.h index ff59924..44f7ba5 100644 --- a/libisofs/node.h +++ b/libisofs/node.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2007 Vreixo Formoso - * Copyright (c) 2009 - 2016 Thomas Schmitt + * Copyright (c) 2009 - 2020 Thomas Schmitt * * This file is part of the libisofs project; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 @@ -437,9 +437,10 @@ int zisofs_zf_xinfo_func(void *data, int flag); * Parameter structure which is to be managed by zisofs_zf_xinfo_func. */ struct zisofs_zf_info { - uint32_t uncompressed_size; + uint64_t uncompressed_size; uint8_t header_size_div4; uint8_t block_size_log2; + uint8_t zisofs_algo[2]; }; /** diff --git a/libisofs/rockridge.c b/libisofs/rockridge.c index c6ef74b..a7da5d9 100644 --- a/libisofs/rockridge.c +++ b/libisofs/rockridge.c @@ -1,7 +1,7 @@ /* * Copyright (c) 2007 Vreixo Formoso * Copyright (c) 2007 Mario Danic - * Copyright (c) 2009 - 2015 Thomas Schmitt + * Copyright (c) 2009 - 2020 Thomas Schmitt * * This file is part of the libisofs project; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 @@ -956,8 +956,8 @@ int pseudo_susp_add_PAD(Ecma119Image *t, struct susp_info *susp) */ static int zisofs_add_ZF(Ecma119Image *t, struct susp_info *susp, int to_ce, - int header_size_div4, int block_size_log2, - uint32_t uncompressed_size, int flag) + uint8_t algo[2], int header_size_div4, int block_size_log2, + uint64_t uncompressed_size, int flag) { unsigned char *ZF = malloc(16); @@ -967,12 +967,21 @@ int zisofs_add_ZF(Ecma119Image *t, struct susp_info *susp, int to_ce, ZF[0] = 'Z'; ZF[1] = 'F'; ZF[2] = (unsigned char) 16; - ZF[3] = (unsigned char) 1; - ZF[4] = (unsigned char) 'p'; - ZF[5] = (unsigned char) 'z'; + if (algo[0] == 'p' && algo[1] == 'z') + ZF[3] = (unsigned char) 1; + else + ZF[3] = (unsigned char) 2; + ZF[4] = (unsigned char) algo[0]; + ZF[5] = (unsigned char) algo[1]; ZF[6] = (unsigned char) header_size_div4; ZF[7] = (unsigned char) block_size_log2; - iso_bb(&ZF[8], uncompressed_size, 4); + if (algo[0] == 'p' && algo[1] == 'z') { + if (uncompressed_size > (uint64_t) 0xffffffff) + return ISO_ZISOFS_TOO_LARGE; + iso_bb(&ZF[8], (uint32_t) uncompressed_size, 4); + } else { + iso_lsb64(&ZF[8], uncompressed_size); + } if (to_ce) { return susp_append_ce(t, susp, ZF); } else { @@ -991,17 +1000,19 @@ int add_zf_field(Ecma119Image *t, Ecma119Node *n, struct susp_info *info, { int ret, will_copy = 1, stream_type = 0, do_zf = 0; int header_size_div4 = 0, block_size_log2 = 0; - uint32_t uncompressed_size = 0; + uint64_t uncompressed_size = 0; IsoStream *stream = NULL, *input_stream, *last_stream, *first_stream; IsoStream *first_filter = NULL; IsoFile *file; void *xipt; struct zisofs_zf_info *zf; + uint8_t algo[2]; /* Intimate friendship with this function in filters/zisofs.c */ int ziso_is_zisofs_stream(IsoStream *stream, int *stream_type, + uint8_t zisofs_algo[2], int *header_size_div4, int *block_size_log2, - uint32_t *uncompressed_size, int flag); + uint64_t *uncompressed_size, int flag); if (!(flag & 1)) flag |= 2; @@ -1043,7 +1054,8 @@ int add_zf_field(Ecma119Image *t, Ecma119Node *n, struct susp_info *info, } /* Determine stream type : 1=ziso , -1=osiz , 0=other */ - ret = ziso_is_zisofs_stream(stream, &stream_type, &header_size_div4, + algo[0] = algo[1] = 0; + ret = ziso_is_zisofs_stream(stream, &stream_type, algo, &header_size_div4, &block_size_log2, &uncompressed_size, 0); if (ret < 0) return ret; @@ -1054,7 +1066,7 @@ int add_zf_field(Ecma119Image *t, Ecma119Node *n, struct susp_info *info, do_zf = 1; } else if(first_stream == last_stream || !will_copy) { /* Try whether the image side stream remembers a ZF field */ - ret = iso_stream_get_src_zf(first_stream, &header_size_div4, + ret = iso_stream_get_src_zf(first_stream, algo, &header_size_div4, &block_size_log2, &uncompressed_size, 0); if (ret == 1 && header_size_div4 > 0) do_zf = 1; @@ -1068,6 +1080,8 @@ int add_zf_field(Ecma119Image *t, Ecma119Node *n, struct susp_info *info, header_size_div4 = zf->header_size_div4; block_size_log2 = zf->block_size_log2; uncompressed_size = zf->uncompressed_size; + algo[0] = zf->zisofs_algo[0]; + algo[1] = zf->zisofs_algo[1]; if (header_size_div4 > 0) do_zf = 1; } @@ -1086,7 +1100,7 @@ int add_zf_field(Ecma119Image *t, Ecma119Node *n, struct susp_info *info, return 1; /* write ZF field */ - ret = zisofs_add_ZF(t, info, (*ce_len > 0), header_size_div4, + ret = zisofs_add_ZF(t, info, (*ce_len > 0), algo, header_size_div4, block_size_log2, uncompressed_size, 0); if (ret < 0) return ret; diff --git a/libisofs/rockridge.h b/libisofs/rockridge.h index 9f3fabd..c2dbed4 100644 --- a/libisofs/rockridge.h +++ b/libisofs/rockridge.h @@ -1,7 +1,7 @@ /* * Copyright (c) 2007 Vreixo Formoso * Copyright (c) 2007 Mario Danic - * Copyright (c) 2009 - 2015 Thomas Schmitt + * Copyright (c) 2009 - 2020 Thomas Schmitt * * This file is part of the libisofs project; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 @@ -359,7 +359,7 @@ int read_aaip_AL(struct susp_sys_user_entry *sue, */ int read_zisofs_ZF(struct susp_sys_user_entry *zf, uint8_t algorithm[2], uint8_t *header_size_div4, uint8_t *block_size_log2, - uint32_t *uncompressed_size, int flag); + uint64_t *uncompressed_size, int flag); /** * Convert a RR filename to the requested charset. diff --git a/libisofs/rockridge_read.c b/libisofs/rockridge_read.c index cc1b70c..9741799 100644 --- a/libisofs/rockridge_read.c +++ b/libisofs/rockridge_read.c @@ -624,7 +624,7 @@ int read_aaip_AL(struct susp_sys_user_entry *sue, */ int read_zisofs_ZF(struct susp_sys_user_entry *zf, uint8_t algorithm[2], uint8_t *header_size_div4, uint8_t *block_size_log2, - uint32_t *uncompressed_size, int flag) + uint64_t *uncompressed_size, int flag) { if (zf == NULL) { return ISO_NULL_POINTER; @@ -635,11 +635,17 @@ int read_zisofs_ZF(struct susp_sys_user_entry *zf, uint8_t algorithm[2], if (zf->len_sue[0] != 16) { return ISO_WRONG_RR; } + if (zf->version[0] > 2) + return ISO_WRONG_RR; algorithm[0] = zf->data.ZF.parameters[0]; algorithm[1] = zf->data.ZF.parameters[1]; *header_size_div4 = zf->data.ZF.parameters[2]; *block_size_log2 = zf->data.ZF.parameters[3]; - *uncompressed_size = iso_read_bb(&(zf->data.ZF.parameters[4]), 4, NULL); + if (zf->version[0] == 1) + *uncompressed_size = iso_read_bb(&(zf->data.ZF.parameters[4]), 4, + NULL); + else + *uncompressed_size = iso_read_lsb64(&(zf->data.ZF.parameters[4])); return ISO_SUCCESS; } diff --git a/libisofs/stream.c b/libisofs/stream.c index ff23595..de22eb6 100644 --- a/libisofs/stream.c +++ b/libisofs/stream.c @@ -290,25 +290,25 @@ int iso_file_source_stream_new(IsoFileSource *src, IsoStream **stream) } -int iso_stream_get_src_zf(IsoStream *stream, int *header_size_div4, - int *block_size_log2, uint32_t *uncompressed_size, - int flag) +int iso_stream_get_src_zf(IsoStream *stream, uint8_t zisofs_algo[2], + int *header_size_div4, int *block_size_log2, + uint64_t *uncompressed_size, int flag) { int ret; FSrcStreamData *data; IsoFileSource *src; /* Intimate friendship with libisofs/fs_image.c */ - int iso_ifs_source_get_zf(IsoFileSource *src, int *header_size_div4, - int *block_size_log2, uint32_t *uncompressed_size, int flag); + int iso_ifs_source_get_zf(IsoFileSource *src, uint8_t zisofs_algo[2], + int *header_size_div4, int *block_size_log2, + uint64_t *uncompressed_size, int flag); if (stream->class != &fsrc_stream_class) return 0; data = stream->data; src = data->src; - - ret = iso_ifs_source_get_zf(src, header_size_div4, block_size_log2, - uncompressed_size, 0); + ret = iso_ifs_source_get_zf(src, zisofs_algo, header_size_div4, + block_size_log2, uncompressed_size, 0); return ret; } diff --git a/libisofs/stream.h b/libisofs/stream.h index 31ee039..0bea3a7 100644 --- a/libisofs/stream.h +++ b/libisofs/stream.h @@ -65,9 +65,9 @@ int iso_cut_out_stream_new(IsoFileSource *src, off_t offset, off_t size, * type, though, unless fsrc_stream_class would be used without FSrcStreamData. * @return 1= returned parameters are valid, 0=no ZF info found , <0 error */ -int iso_stream_get_src_zf(IsoStream *stream, int *header_size_div4, - int *block_size_log2, uint32_t *uncompressed_size, - int flag); +int iso_stream_get_src_zf(IsoStream *stream, uint8_t zisofs_algo[2], + int *header_size_div4, int *block_size_log2, + uint64_t *uncompressed_size, int flag); /** * Set the inode number of a stream that is based on FSrcStreamData, i.e. diff --git a/libisofs/util.c b/libisofs/util.c index 5de3c9f..e344656 100644 --- a/libisofs/util.c +++ b/libisofs/util.c @@ -1483,6 +1483,14 @@ void iso_lsb(uint8_t *buf, uint32_t num, int bytes) buf[i] = (num >> (8 * i)) & 0xff; } +void iso_lsb64(uint8_t *buf, uint64_t num) +{ + int i; + + for (i = 0; i < 8; ++i) + buf[i] = (num >> (8 * i)) & 0xff; +} + void iso_msb(uint8_t *buf, uint32_t num, int bytes) { int i; diff --git a/libisofs/util.h b/libisofs/util.h index dfd116d..6f32176 100644 --- a/libisofs/util.h +++ b/libisofs/util.h @@ -264,6 +264,7 @@ int str2d_char(const char *icharset, const char *input, char **output); int str2a_char(const char *icharset, const char *input, char **output); void iso_lsb(uint8_t *buf, uint32_t num, int bytes); +void iso_lsb64(uint8_t *buf, uint64_t num); void iso_msb(uint8_t *buf, uint32_t num, int bytes); void iso_bb(uint8_t *buf, uint32_t num, int bytes);