From c1e6d3415a68c3a5a62a953758c41b7d4cdc07ea Mon Sep 17 00:00:00 2001 From: Thomas Schmitt Date: Sun, 25 Oct 2020 17:04:24 +0100 Subject: [PATCH] In case of ISO_ZISOFS_TOO_MANY_PTR remove all dispensable block pointers --- xorriso/base_obj.c | 2 +- xorriso/filters.c | 19 +++++ xorriso/xorriso.1 | 21 ++++-- xorriso/xorriso.info | 141 +++++++++++++++++++----------------- xorriso/xorriso.texi | 21 ++++-- xorriso/xorriso_timestamp.h | 2 +- 6 files changed, 120 insertions(+), 86 deletions(-) diff --git a/xorriso/base_obj.c b/xorriso/base_obj.c index 3994c6f9..b8c9edd0 100644 --- a/xorriso/base_obj.c +++ b/xorriso/base_obj.c @@ -202,7 +202,7 @@ int Xorriso_new(struct XorrisO ** xorriso,char *progname, int flag) m->zisofs_v2_block_size= m->zisofs_v2_block_size_default= (1 << 17); m->zisofs_block_number_target= -1; m->zisofs_bpt_discard_free_ratio= m->zisofs_bpt_discard_free_ratio_default= - 0.5; + -1.0; m->do_overwrite= 2; m->do_reassure= 0; m->drive_blacklist= NULL; diff --git a/xorriso/filters.c b/xorriso/filters.c index 3b68eab9..c43b545c 100644 --- a/xorriso/filters.c +++ b/xorriso/filters.c @@ -378,6 +378,25 @@ int Xorriso_set_filter(struct XorrisO *xorriso, void *in_node, filter_ret= 1; } else if (internal_filter == 1 || internal_filter == 2) { filter_ret = iso_file_add_zisofs_filter(file, 1 | (internal_filter & 2)); + if(filter_ret == (int) ISO_ZISOFS_TOO_MANY_PTR) { + /* Remove all buffered currently unused block pointers and try again */ + ret= Xorriso_get_volume(xorriso, &volume, 0); + if(ret < 0) + goto ex; + if(ret > 0) { + Xorriso_msgs_submit(xorriso, 0, + "Overflow of zisofs block pointers happened.", + 0, "WARNING", 0); + Xorriso_msgs_submit(xorriso, 0, + "zisofs falling back to mode which needs 3 input read runs.", + 0, "WARNING", 0); + ret = iso_image_zisofs_discard_bpt(volume, 0); + if(ret <= 0) + goto ex; + filter_ret = iso_file_add_zisofs_filter(file, + 1 | (internal_filter & 2)); + } + } if(filter_ret < 0) { Xorriso_process_msg_queues(xorriso,0); if(!(internal_filter == 2 && filter_ret == (int) ISO_ZISOFS_WRONG_INPUT)) diff --git a/xorriso/xorriso.1 b/xorriso/xorriso.1 index 3ba739f3..60491c91 100644 --- a/xorriso/xorriso.1 +++ b/xorriso/xorriso.1 @@ -9,7 +9,7 @@ .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) -.TH XORRISO 1 "Version 1.5.3, Oct 21, 2020" +.TH XORRISO 1 "Version 1.5.3, Oct 25, 2020" .\" Please adjust this date whenever revising the manpage. .\" .\" Some roff macros, for reference: @@ -3141,7 +3141,7 @@ pointers if block size is 32k, and 2 block pointers with block size 64k. .br bpt_target=\-1 disables this automatic block size adjustment. .br - "max_bpt="1k|...|128g sets the limit for the overall allocated block pointer + "max_bpt="1k...128g sets the limit for the overall allocated block pointer memory. Block pointers occupy virtual memory while a file gets uncompressed and while a file, which shall be compressed, waits for ISO filesystem creation. .br @@ -3151,22 +3151,27 @@ suffices for at most 16g of uncompressed file size. Each file consumes one end block pointer, independently of the file size. Partially filled end blocks may further reduce the effective payload. .br - "max_bpt_f="1k|...|128g sets the limit for the memory size of the block +In case of overflow of the max_bpt limit while adding compression filters +the program tries to go on by discarding all buffered block pointers of +previously added \-\-zisofs filters. From then on all newly added filters will +discard their block pointers immediately after being added. +Discarded block pointers cause an additional read and compression run of the +input file during the production of the ISO filesystem. +.br + "max_bpt_f="1k...128g sets the limit for the memory size of the block pointer list of a single file. max_bpt_f is never larger than max_bpt. If either is set to violate this rule, the other gets set to the same value. If both values are the same before a change by max_bpt= or max_bpt_f=, then both limits stick together unless the limit is decreased by max_bpt_f=. .br - "bpt_free_ratio="\-1|[0.0...1.0] sets a threshold for switching to block + "bpt_free_ratio="\-1|0.0...1.0 sets a threshold for switching to block pointer discarding during compression. If less than the given fraction of the max_bpt_f= memory is free, then block pointers of compression filters get -discarded after size assessment. This causes a third read run of the input -file during the production of the ISO filesystem. Value \-1 disables this -feature. +discarded immediately after being added. Value \-1 disables this feature. .br "default" is the same as "level=6:block_size=32k:by_magic=off: version_2=off:block_size_v2=128k:max_bpt=256m:max_bpt_f=256m: -bpt_free_ratio=0.5". +bpt_free_ratio=\-1". .TP \fB\-speed\fR code|number[k|m|c|d|b] Set the burn speed. Default is "max" (or "0") = maximum speed as announced diff --git a/xorriso/xorriso.info b/xorriso/xorriso.info index f8d1888b..6b631fd8 100644 --- a/xorriso/xorriso.info +++ b/xorriso/xorriso.info @@ -2670,9 +2670,9 @@ according to the setting of command -acl. has 3 block pointers if block size is 32k, and 2 block pointers with block size 64k. bpt_target=-1 disables this automatic block size adjustment. - "max_bpt="1k|...|128g sets the limit for the overall allocated - block pointer memory. Block pointers occupy virtual memory while a - file gets uncompressed and while a file, which shall be compressed, + "max_bpt="1k...128g sets the limit for the overall allocated block + pointer memory. Block pointers occupy virtual memory while a file + gets uncompressed and while a file, which shall be compressed, waits for ISO filesystem creation. One pointer occupies 8 bytes of memory and governs block_size or block_size_v2 uncompressed bytes. I.e. with block size 128k, 1m @@ -2680,22 +2680,27 @@ according to the setting of command -acl. file size. Each file consumes one end block pointer, independently of the file size. Partially filled end blocks may further reduce the effective payload. - "max_bpt_f="1k|...|128g sets the limit for the memory size of the + In case of overflow of the max_bpt limit while adding compression + filters the program tries to go on by discarding all buffered block + pointers of previously added -zisofs filters. From then on all + newly added filters will discard their block pointers immediately + after being added. Discarded block pointers cause an additional + read and compression run of the input file during the production of + the ISO filesystem. + "max_bpt_f="1k...128g sets the limit for the memory size of the block pointer list of a single file. max_bpt_f is never larger than max_bpt. If either is set to violate this rule, the other gets set to the same value. If both values are the same before a change by max_bpt= or max_bpt_f=, then both limits stick together unless the limit is decreased by max_bpt_f=. - "bpt_free_ratio="-1|[0.0...1.0] sets a threshold for switching to + "bpt_free_ratio="-1|0.0...1.0 sets a threshold for switching to block pointer discarding during compression. If less than the given fraction of the max_bpt_f= memory is free, then block - pointers of compression filters get discarded after size - assessment. This causes a third read run of the input file during - the production of the ISO filesystem. Value -1 disables this - feature. + pointers of compression filters get discarded immediately after + being added. Value -1 disables this feature. "default" is the same as "level=6:block_size=32k:by_magic=off: version_2=off:block_size_v2=128k:max_bpt=256m:max_bpt_f=256m: - bpt_free_ratio=0.5". + bpt_free_ratio=-1". -speed code|number[k|m|c|d|b] Set the burn speed. Default is "max" (or "0") = maximum speed as announced by the drive. Further special speed codes are: @@ -5500,7 +5505,7 @@ File: xorriso.info, Node: CommandIdx, Next: ConceptIdx, Prev: Legal, Up: Top * -chown sets ownership in ISO image: Manip. (line 43) * -chown_r sets ownership in ISO image: Manip. (line 47) * -clone copies ISO directory tree: Insert. (line 185) -* -close controls media closing: SetWrite. (line 461) +* -close controls media closing: SetWrite. (line 466) * -close_damaged closes damaged track and session: Writing. (line 205) * -close_filter_list bans filter registration: Filter. (line 50) * -commit writes pending ISO image: Writing. (line 27) @@ -5530,11 +5535,11 @@ File: xorriso.info, Node: CommandIdx, Next: ConceptIdx, Prev: Legal, Up: Top * -drive_access control device file locking: AqDrive. (line 72) * -drive_class controls drive accessability: AqDrive. (line 43) * -du show directory size in ISO image: Navigate. (line 78) -* -dummy controls write simulation: SetWrite. (line 453) +* -dummy controls write simulation: SetWrite. (line 458) * -dus show directory size in ISO image: Navigate. (line 81) * -dusx show directory size on disk: Navigate. (line 88) * -dux show directory size on disk: Navigate. (line 84) -* -dvd_obs set write block size: SetWrite. (line 390) +* -dvd_obs set write block size: SetWrite. (line 395) * -early_stdio_test classifies stdio drives: Loading. (line 341) * -ecma119_map names w/o Rock Ridge, Joliet: Loading. (line 229) * -eject ejects drive tray: Writing. (line 50) @@ -5554,7 +5559,7 @@ File: xorriso.info, Node: CommandIdx, Next: ConceptIdx, Prev: Legal, Up: Top * -follow softlinks and mount points: SetInsert. (line 69) * -format formats media: Writing. (line 87) * -for_backup -acl,-xattr,-hardlinks,-md5: Loading. (line 215) -* -fs sets size of fifo: SetWrite. (line 456) +* -fs sets size of fifo: SetWrite. (line 461) * -getfacl shows ACL in ISO image: Navigate. (line 60) * -getfacl_r shows ACL in ISO image: Navigate. (line 66) * -getfattr shows xattr in ISO image: Navigate. (line 69) @@ -5599,7 +5604,7 @@ File: xorriso.info, Node: CommandIdx, Next: ConceptIdx, Prev: Legal, Up: Top * -mark sets synchronizing message: Frontend. (line 23) * -md5 controls handling of MD5 sums: Loading. (line 184) * -mkdir creates ISO directory: Insert. (line 177) -* -modesty_on_drive keep drive buffer hungry: SetWrite. (line 396) +* -modesty_on_drive keep drive buffer hungry: SetWrite. (line 401) * -mount issues mount command for ISO session: Restore. (line 153) * -mount_cmd composes mount command line: Inquiry. (line 49) * -mount_cmd controls mount command: Inquiry. (line 65) @@ -5618,7 +5623,7 @@ File: xorriso.info, Node: CommandIdx, Next: ConceptIdx, Prev: Legal, Up: Top * -out_charset sets output character set: SetWrite. (line 285) * -overwrite enables overwriting in ISO: SetInsert. (line 131) * -pacifier controls pacifier text form: Emulation. (line 166) -* -padding sets amount or mode of image padding: SetWrite. (line 484) +* -padding sets amount or mode of image padding: SetWrite. (line 489) * -page set terminal geometry: DialogCtl. (line 18) * -paste_in copies file into disk file: Restore. (line 121) * -pathspecs sets meaning of = with -add: SetInsert. (line 115) @@ -5676,12 +5681,12 @@ File: xorriso.info, Node: CommandIdx, Next: ConceptIdx, Prev: Legal, Up: Top * -signal_handling controls handling of system signals: Exception. (line 66) * -sleep waits for a given time span: Scripting. (line 113) -* -speed set write speed: SetWrite. (line 361) +* -speed set write speed: SetWrite. (line 366) * -split_size enables large file splitting: SetInsert. (line 145) * -status shows current settings: Scripting. (line 44) * -status_history_max curbs -status history: Scripting. (line 52) -* -stdio_sync controls stdio buffer: SetWrite. (line 446) -* -stream_recording controls defect management: SetWrite. (line 379) +* -stdio_sync controls stdio buffer: SetWrite. (line 451) +* -stream_recording controls defect management: SetWrite. (line 384) * -system_id sets system id: SetWrite. (line 205) * -tell_media_space reports free space: Inquiry. (line 98) * -temp_mem_limit curbs memory consumption: Scripting. (line 96) @@ -5694,13 +5699,13 @@ File: xorriso.info, Node: CommandIdx, Next: ConceptIdx, Prev: Legal, Up: Top * -update_l inserts paths if different <1>: Insert. (line 128) * -update_li inserts paths if different: Insert. (line 124) * -update_r inserts paths if different: Insert. (line 110) -* -use_immed_bit controls use of Immed bit: SetWrite. (line 434) +* -use_immed_bit controls use of Immed bit: SetWrite. (line 439) * -use_readline enables readline for dialog: DialogCtl. (line 26) * -version prints help text: Scripting. (line 22) * -volid sets volume id: SetWrite. (line 168) * -volset_id sets volume set id: SetWrite. (line 188) * -volume_date sets volume timestamp: SetWrite. (line 211) -* -write_type chooses TAO or SAO/DAO: SetWrite. (line 477) +* -write_type chooses TAO or SAO/DAO: SetWrite. (line 482) * -x enables automatic execution order of arguments: ArgSort. (line 16) * -xattr controls handling of xattr (EA): Loading. (line 172) * -zisofs controls zisofs production: SetWrite. (line 296) @@ -5733,7 +5738,7 @@ File: xorriso.info, Node: ConceptIdx, Prev: CommandIdx, Up: Top * Backup, enable features, -for_backup: Loading. (line 215) * Backup, scdbackup checksum tag, -scdbackup: Emulation. (line 179) * Blank media, _definition: Media. (line 29) -* Blank, format, Immed bit, -use_immed_bit: SetWrite. (line 434) +* Blank, format, Immed bit, -use_immed_bit: SetWrite. (line 439) * Blind growing, _definition: Methods. (line 41) * Bootability, control, -boot_image: Bootable. (line 75) * Bugs, reporting: Bugreport. (line 6) @@ -5996,26 +6001,26 @@ File: xorriso.info, Node: ConceptIdx, Prev: CommandIdx, Up: Top * Verify, file checksum, -check_md5: Verify. (line 184) * Verify, file tree checksums, -check_md5_r: Verify. (line 198) * Verify, preset -check_media, -check_media_defaults: Verify. (line 40) -* Write, block size, -dvd_obs: SetWrite. (line 390) +* Write, block size, -dvd_obs: SetWrite. (line 395) * Write, bootability, -boot_image: Bootable. (line 75) -* Write, buffer syncing, -stdio_sync: SetWrite. (line 446) -* Write, close media, -close: SetWrite. (line 461) +* Write, buffer syncing, -stdio_sync: SetWrite. (line 451) +* Write, close media, -close: SetWrite. (line 466) * Write, compliance to specs, -compliance: SetWrite. (line 62) -* Write, defect management, -stream_recording: SetWrite. (line 379) +* Write, defect management, -stream_recording: SetWrite. (line 384) * Write, disable Rock Ridge, -rockridge: SetWrite. (line 57) -* Write, drive buffer, -modesty_on_drive: SetWrite. (line 396) +* Write, drive buffer, -modesty_on_drive: SetWrite. (line 401) * Write, enable HFS+, -hfsplus: SetWrite. (line 14) * Write, enable Joliet, -joliet: SetWrite. (line 10) -* Write, fifo size, -fs: SetWrite. (line 456) +* Write, fifo size, -fs: SetWrite. (line 461) * Write, free space, -tell_media_space: Inquiry. (line 98) * Write, log problematic disk files, -errfile_log: Scripting. (line 116) * Write, log written sessions, -session_log: Scripting. (line 134) -* Write, padding image, -padding: SetWrite. (line 484) +* Write, padding image, -padding: SetWrite. (line 489) * Write, pending ISO image, -commit: Writing. (line 27) * Write, predict image size, -print_size: Inquiry. (line 86) -* Write, set speed, -speed: SetWrite. (line 361) -* Write, simulation, -dummy: SetWrite. (line 453) -* Write, TAO or SAO/DAO, -write_type: SetWrite. (line 477) +* Write, set speed, -speed: SetWrite. (line 366) +* Write, simulation, -dummy: SetWrite. (line 458) +* Write, TAO or SAO/DAO, -write_type: SetWrite. (line 482) * xattr, control handling, -xattr: Loading. (line 172) * xattr, set in ISO image, -setfattr: Manip. (line 103) * xattr, set in ISO image, -setfattr_list: Manip. (line 120) @@ -6047,41 +6052,41 @@ Node: CmdFind88338 Node: Filter107267 Node: Writing111889 Node: SetWrite124144 -Node: Bootable152434 -Node: Jigdo179519 -Node: Charset184522 -Node: Exception187851 -Node: DialogCtl193980 -Node: Inquiry196582 -Node: Navigate205464 -Node: Verify214171 -Node: Restore225320 -Node: Emulation234486 -Node: Scripting244942 -Node: Frontend252725 -Node: Examples262351 -Node: ExDevices263529 -Node: ExCreate264190 -Node: ExDialog265490 -Node: ExGrowing266761 -Node: ExModifying267570 -Node: ExBootable268080 -Node: ExCharset268635 -Node: ExPseudo269531 -Node: ExCdrecord270458 -Node: ExMkisofs270778 -Node: ExGrowisofs272675 -Node: ExException273828 -Node: ExTime274286 -Node: ExIncBackup274744 -Node: ExRestore278770 -Node: ExRecovery279716 -Node: Files280288 -Node: Environ281622 -Node: Seealso282370 -Node: Bugreport283087 -Node: Legal283678 -Node: CommandIdx284690 -Node: ConceptIdx302306 +Node: Bootable152784 +Node: Jigdo179869 +Node: Charset184872 +Node: Exception188201 +Node: DialogCtl194330 +Node: Inquiry196932 +Node: Navigate205814 +Node: Verify214521 +Node: Restore225670 +Node: Emulation234836 +Node: Scripting245292 +Node: Frontend253075 +Node: Examples262701 +Node: ExDevices263879 +Node: ExCreate264540 +Node: ExDialog265840 +Node: ExGrowing267111 +Node: ExModifying267920 +Node: ExBootable268430 +Node: ExCharset268985 +Node: ExPseudo269881 +Node: ExCdrecord270808 +Node: ExMkisofs271128 +Node: ExGrowisofs273025 +Node: ExException274178 +Node: ExTime274636 +Node: ExIncBackup275094 +Node: ExRestore279120 +Node: ExRecovery280066 +Node: Files280638 +Node: Environ281972 +Node: Seealso282720 +Node: Bugreport283437 +Node: Legal284028 +Node: CommandIdx285040 +Node: ConceptIdx302656  End Tag Table diff --git a/xorriso/xorriso.texi b/xorriso/xorriso.texi index 00548304..f4965974 100644 --- a/xorriso/xorriso.texi +++ b/xorriso/xorriso.texi @@ -50,7 +50,7 @@ @c man .\" First parameter, NAME, should be all caps @c man .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection @c man .\" other parameters are allowed: see man(7), man(1) -@c man .TH XORRISO 1 "Version 1.5.3, Oct 21, 2020" +@c man .TH XORRISO 1 "Version 1.5.3, Oct 25, 2020" @c man .\" Please adjust this date whenever revising the manpage. @c man .\" @c man .\" Some roff macros, for reference: @@ -3654,7 +3654,7 @@ pointers if block size is 32k, and 2 block pointers with block size 64k. @* bpt_target=-1 disables this automatic block size adjustment. @* - "max_bpt="1k|...|128g sets the limit for the overall allocated block pointer + "max_bpt="1k...128g sets the limit for the overall allocated block pointer memory. Block pointers occupy virtual memory while a file gets uncompressed and while a file, which shall be compressed, waits for ISO filesystem creation. @* @@ -3664,22 +3664,27 @@ suffices for at most 16g of uncompressed file size. Each file consumes one end block pointer, independently of the file size. Partially filled end blocks may further reduce the effective payload. @* - "max_bpt_f="1k|...|128g sets the limit for the memory size of the block +In case of overflow of the max_bpt limit while adding compression filters +the program tries to go on by discarding all buffered block pointers of +previously added --zisofs filters. From then on all newly added filters will +discard their block pointers immediately after being added. +Discarded block pointers cause an additional read and compression run of the +input file during the production of the ISO filesystem. +@* + "max_bpt_f="1k...128g sets the limit for the memory size of the block pointer list of a single file. max_bpt_f is never larger than max_bpt. If either is set to violate this rule, the other gets set to the same value. If both values are the same before a change by max_bpt= or max_bpt_f=, then both limits stick together unless the limit is decreased by max_bpt_f=. @* - "bpt_free_ratio="-1|[0.0...1.0] sets a threshold for switching to block + "bpt_free_ratio="-1|0.0...1.0 sets a threshold for switching to block pointer discarding during compression. If less than the given fraction of the max_bpt_f= memory is free, then block pointers of compression filters get -discarded after size assessment. This causes a third read run of the input -file during the production of the ISO filesystem. Value -1 disables this -feature. +discarded immediately after being added. Value -1 disables this feature. @* "default" is the same as "level=6:block_size=32k:by_magic=off: version_2=off:block_size_v2=128k:max_bpt=256m:max_bpt_f=256m: -bpt_free_ratio=0.5". +bpt_free_ratio=-1". @c man .TP @item -speed code|number[k|m|c|d|b] @kindex -speed set write speed diff --git a/xorriso/xorriso_timestamp.h b/xorriso/xorriso_timestamp.h index b144d36f..3fb3313d 100644 --- a/xorriso/xorriso_timestamp.h +++ b/xorriso/xorriso_timestamp.h @@ -1 +1 @@ -#define Xorriso_timestamP "2020.10.24.121839" +#define Xorriso_timestamP "2020.10.25.160005"