From 9b4c082f84046d21554e9f747914e39d701ef9cb Mon Sep 17 00:00:00 2001 From: Thomas Schmitt Date: Wed, 15 Jun 2022 12:25:05 +0200 Subject: [PATCH] New experimental script test/merge_debian_isos. Deprecated merge_2_debian_isos. --- test/merge_2_debian_isos | 13 + test/merge_debian_isos | 634 +++++++++++++++++++++++++++++++++++++ test/merge_debian_isos.sig | Bin 0 -> 72 bytes 3 files changed, 647 insertions(+) create mode 100755 test/merge_debian_isos create mode 100644 test/merge_debian_isos.sig diff --git a/test/merge_2_debian_isos b/test/merge_2_debian_isos index dd0f9fe0..da3d7249 100755 --- a/test/merge_2_debian_isos +++ b/test/merge_2_debian_isos @@ -1,5 +1,15 @@ #!/bin/bash +echo >&2 +echo "THIS SCRIPT IS DEPRECATED ! USE ITS SUCCESSOR: merge_debian_isos" >&2 +echo >&2 +echo "The script merge_2_debian_isos still exists only because it was mentioned">&2 +echo "in Debian bug 1011343. The successor can merge more than two ISOs.">&2 +echo "So do not edit this script to remove this warning and the 'exit 7' line.">&2 +echo >&2 + +exit 7 + usage() { echo "usage: $(basename "$0") "'\' >&2 echo " boot_iso boot_mount add_iso add_mount result_iso [for_dist]" >&2 @@ -344,6 +354,9 @@ fi -padding included \ -compliance no_emul_toc + +## Clean up + if test "$do_mount_1" = 1 then sudo umount "$BOOT_ISO_MOUNT" diff --git a/test/merge_debian_isos b/test/merge_debian_isos new file mode 100755 index 00000000..e3ac3905 --- /dev/null +++ b/test/merge_debian_isos @@ -0,0 +1,634 @@ +#!/bin/bash +# Copyright 2022 Thomas Schmitt , libburnia project. +# Provided under BSD license: Use, modify, and distribute as you like. + +usage() { + echo "usage: $(basename "$0") result_iso mount_template iso1 iso2 [... isoN]" >&2 + echo >&2 + echo "Mounts by sudo the ISO 9660 images iso1 to isoN at directories" >&2 + echo "mount_template1 to mount_templateN, if not already mounted that way." >&2 + echo "Then the Debian pools and package lists get merged and a new" >&2 + echo "ISO 9660 image result_iso is produced, which must not yet exist." >&2 + echo "If iso1 is bootable then the new image will be bootable by the" >&2 + echo "same means." >&2 + echo "At least the parent directory of mount_template must already exist." >&2 + echo "All arguments must be single words without using quotation marks." >&2 + echo "None of the isoN must be equal to another isoM." >&2 + echo >&2 + echo "This script creates and finally removes the following temporary tree" >&2 + echo "and files which must not yet exist in the current working directory:" >&2 + echo " ./merged_dists , ./merged_md5sum.txt , ./merged_REAMDE.txt" >&2 + echo " ./temp_file" >&2 + echo "Further it creates and finally removes directories mount_template*" >&2 + echo "if they are needed and do not exist when the script starts." >&2 + echo "It depends on the following programs:" >&2 + echo " awk, basename, bash, cat, chmod, cp, dirname, expr, fgrep, grep," >&2 + echo " gunzip, gzip, head, ls, mkdir, mount, mv, rm, rmdir, sha256sum," >&2 + echo " sed, sort, stat, sudo, umount, xorriso" >&2 + echo "Recommended are: md5sum, sha1sum, sha512sum" >&2 + echo >&2 + echo "Exported non-empty variable MERGE_DATE enforces a particular" >&2 + echo "date string in the text which gets prepended to /README.txt ." >&2 + echo "Exported non-empty variable MERGE_FOR_DIST enforces the use of a" >&2 + echo "particular directory in /dists of iso1. Normally only one" >&2 + echo "such directory is found and thus no need to set MERGE_FOR_DIST." >&2 + echo "Exported non-empty variable XORRISO overrides command xorriso." >&2 + echo "This may be needed if installed xorriso is older than 1.4.2." >&2 + echo >&2 + echo "Example using GNU xorriso-1.5.4 instead of /usr/bin/xorriso:" >&2 + echo " export XORRISO="'$HOME'"/xorriso-1.5.4/xorriso/xorriso" >&2 + echo " mkdir merge_mount" >&2 + echo " $(basename "$0") merged.iso merge_mount/iso "'\' >&2 + echo " debian-11.2.0-amd64-DVD-[12345].iso" >&2 + echo " rmdir merge_mount" >&2 +} + +check_single_word() { + empty=1 + for i in $1 + do + if test "$i" = "$1" + then + empty=0 + else + echo "--- Argument $2 is not a single word:" >&2 + echo "--- '${1}'" >&2 + return 1 + fi + done + if test "$empty" = 1 + then + echo "--- Argument $2 is empty or entirely white space:" >&2 + echo "--- '${1}'" >&2 + return 1 + fi + return 0 +} + +MOUNT_LIST= +UMOUNT_LIST= +RMDIR_LIST= +TEMPFILE_LIST="merged_dists merged_md5sum.txt merged_README.txt temp_file" + +# Cleanup temporary files, mount points and made directories +cleanup_and_end() { + # Remove trap + trap EXIT + + echo >&2 + echo "Cleaning up temporary files and mount points ..." 2>&1 + + ret=0 + + for i in $TEMPFILE_LIST + do + if test -e "$i" + then + if rm -r "$i" + then + dummy=dummy + else + echo "--- Note: Cannot remove previously created temporary file: $i" >&2 + ret=1 + fi + fi + done + + for i in $UMOUNT_LIST + do + if sudo umount "$i" + then + dummy=dummy + else + echo "--- Note: Cannot unmount previously mounted $i" >&2 + ret=1 + fi + done + + for i in $RMDIR_LIST + do + if rmdir "$i" + then + dummy=dummy + else + echo "--- Note: Cannot remove previously created directory $i" >&2 + ret=1 + fi + done + if test "$ret" = 0 + then + echo "Cleanup completed." >&2 + else + echo "--- Cleanup could not be fully completed." >&2 + fi + + if test -n "$1" + then + echo >&2 + if test "$1" -gt 0 + then + echo "--- Merge run aborted !" >&2 + else + echo "Merge run ended with success indication." >&2 + fi + exit "$1" + fi + + return $ret +} + + +## Check arguments + +# +++ No cleanup_and_end before TEMPFILE_LIST is verified to be missing +++ + +if test "$#" -lt 4 +then + usage + exit 1 +fi + +echo >&2 +echo "$(basename "$0") starting with $(expr $# - 2) ISO image files ..." >&2 + +existing_tempfiles= +for i in $TEMPFILE_LIST +do + if test -e "$i" + then + existing_tempfiles="$existing_tempfiles $i" + fi +done +if test -n "$existing_tempfiles" +then + echo "--- Some temporary files for this script already exist:" >&2 + echo "--- $existing_tempfiles" >&2 + echo "--- Will not overwrite them." >&2 + echo "--- Merge run aborted !" >&2 + exit 1 +fi + +# +++ From here on: Always call cleanup_and_end to perform exit +++ +trap cleanup_and_end EXIT + +RESULT_ISO="$1" +check_single_word "$2" "result_iso" || cleanup_and_end 1 +if test -e "$RESULT_ISO" +then + echo "--- A file '${RESULT_ISO}' is already existing." >&2 + echo "--- Will not overwrite it by the resulting ISO image." >&2 + cleanup_and_end 1 +fi + +MOUNT_TEMPLATE="$2" +check_single_word "$1" "mount_template" || cleanup_and_end 1 +x=$(dirname "$MOUNT_TEMPLATE") +if test -d "$x" +then + dummy=dummy +else + echo "--- The parent directory of '${MOUNT_TEMPLATE}' does not exist." >&2 + cleanup_and_end 1 +fi + +shift 2 +ISO_LIST= +mount_count=0 +for i in "$@" +do + mount_count=$(expr $mount_count + 1) + + check_single_word "$i" "iso$mount_count" || cleanup_and_end 1 + + if test "$i" = "$RESULT_ISO" + then + echo "--- Arguments result_iso and iso$mount_count are equal:" >&2 + echo "--- '${i}'" >&2 + cleanup_and_end 1 + fi + + if echo "$ISO_LIST" | fgrep " $i " >/dev/null + then + echo "--- Duplicate file path given as argument iso$mount_count :" >&2 + echo "--- '${i}'" >&2 + cleanup_and_end 1 + fi + + x="${MOUNT_TEMPLATE}$mount_count" + if test -d "$x" + then + dummy=dummy + elif test -e "$x" + then + echo "--- A file '${x}' is already existing and not a directory." >&2 + echo "--- Cannot mount iso$mount_count ('${i}')" >&2 + cleanup_and_end 1 + fi + + ISO_LIST="$ISO_LIST $i " + if test -z "$iso_1" + then + iso_1="$i" + fi +done +echo "Arguments look acceptable." >&2 + + +## Mount and copy out the files which need to be changed +echo >&2 +echo "Mounting ISO images if not yet mounted ..." >&2 + +mount_count=0 +for i in $ISO_LIST +do + mount_count=$(expr $mount_count + 1) + mount_point="${MOUNT_TEMPLATE}$mount_count" + if test -d "$mount_point" + then + dummy=dummy + else + if mkdir "$mount_point" + then + RMDIR_LIST="$RMDIR_LIST $mount_point " + else + echo "--- Could not create directory '${mount_point}'." >&2 + echo "--- Cannot mount iso$mount_count ('${i}')" >&2 + cleanup_and_end 3 + fi + fi + + do_mount=1 + if echo "$mount_point" | grep '^/' >/dev/null + then + x=$(mount | grep " $mount_point " | awk '{print $1}') + elif echo "$mount_point" | grep '^./' >/dev/null + then + m=$(echo "$mount_point" | sed -e 's/^\.\///') + x=$(mount | grep " $(pwd)/$m" | awk '{print $1}') + else + x=$(mount | grep " $(pwd)/$mount_point " | awk '{print $1}') + fi + if test -n "$x" + then + i1=$(ls -i "$x" | awk '{print $1}') + i2=$(ls -i "$i" | awk '{print $1}') + if test "$i1" = "$i2" + then + do_mount=0 + echo "Note: Found $i already mounted at $mount_point" + fi + fi + if test "$do_mount" = 1 + then + echo "Note: sudo mount $i $mount_point" + if sudo mount "$i" "$mount_point" + then + dummy=dummy + else + echo "--- Could not mount '${i}' at '${mount_point}'." >&2 + cleanup_and_end 3 + fi + UMOUNT_LIST="$UMOUNT_LIST $mount_point" + fi + + MOUNT_LIST="$MOUNT_LIST $mount_point" +done + +echo >&2 +echo "Copying dists directory and md5sum.txt from first ISO ..." >&2 + +mount_point_1="$MOUNT_TEMPLATE"1 +if test -d "$mount_point_1/dists" +then + echo "Copying: $mount_point_1/dists to merged_dists" >&2 + if cp -a "$mount_point_1/dists" merged_dists + then + if chmod -R u+w merged_dists + then + dummy=dummy + else + echo "--- Could not chmod -R u+w merged_dists" >&2 + cleanup_and_end 3 + fi + else + echo "--- Could not copy /dists directory from first ISO." >&2 + cleanup_and_end 3 + fi +else + echo "--- First ISO does not contain a /dists directory." >&2 + cleanup_and_end 2 +fi +echo "Copying: $mount_point_1/md5sum.txt to merged_md5sum.txt" >&2 +if cp -a "$mount_point_1/md5sum.txt" merged_md5sum.txt +then + if chmod u+w merged_md5sum.txt + then + dummy=dummy + else + echo "--- Could not chmod u+w merged_md5sum.txt" >&2 + cleanup_and_end 3 + fi +else + echo "--- Could not copy /md5sum.txt from first ISO." >&2 + cleanup_and_end 3 +fi + + +## Helper functions + +# Put out the list of checksummed paths as listed in /dists/$dist/Release +extract_checksum_paths() { + mode=0 + cat "$1" | \ + while true + do + read x || break + if test "$x" = "MD5Sum:" || test "$x" = "SHA1:" \ + || test "$x" = "SHA256:" || test "$x" = "SHA512:" + then + if test "$mode" = 0 + then + mode=1 + elif test "$mode" = 1 + then + break + fi + elif test "$mode" = 1 + then + echo "$x" + fi + done +} + +# Put out the part before the first checksum field +extract_release_head() { + cat "$1" | \ + while true + do + read x || break + if test "$x" = "MD5Sum:" || test "$x" = "SHA1:" \ + || test "$x" = "SHA256:" || test "$x" = "SHA512:" + then + break + fi + echo "$x" + done +} + + +## Determine which Debian release is on iso1 +echo >&2 +echo "Determining Debian release in first ISO ..." >&2 + +dist= +for i in $(ls -1d "$mount_point_1"/dists/*) +do + if test -d "$i" + then + if test -L "$i" + then +continue + fi + test -n "$dist" && dist="$dist " + dist="${dist}$(basename $i)" + fi +done +if test -z "$dist" +then + if test -z "$MERGE_FOR_DIST" + then + echo "--- Cannot determine Debian release from directories in /dists" >&2 + echo "--- (You may provide the release name as variable MERGE_FOR_DIST)" >&2 + echo >&2 + cleanup_and_end 2 + fi +elif test "$(echo "$dist" | wc -w)" -gt 1 +then + if test -z "$MERGE_FOR_DIST" + then + echo "--- More than one Debian release found in /dists: $dist" >&2 + echo "--- (You may provide the release name as variable MERGE_FOR_DIST)" >&2 + echo >&2 + cleanup_and_end 2 + fi +fi +if test -n "$MERGE_FOR_DIST" +then + echo "Note: Overriding release name '${dist}' by '${MERGE_FOR_DIST}'" >&2 + dist="$MERGE_FOR_DIST" +fi +if test -d "$mount_point_1"/dists/"$dist" +then + echo "Will work along $mount_point_1"/dists/"$dist"/Release >&2 +else + echo "--- Cannot find directory $mount_point_1"/dists/"$dist" >&2 + cleanup_and_end 2 +fi +for i in $MOUNT_LIST +do + if test -e "$i"/dists/"$dist"/Release + then + dummy=dummy + else + echo "--- Cannot find file $i"/dists/"$dist"/Release >&2 + echo "--- All participating ISOs must be installation ISOs of the same release." >&2 + cleanup_and_end 2 + fi +done + + +## Prepend info to /README.txt +echo >&2 +echo "Composing new /README.txt ..." >&2 + +if test -z "$MERGE_DATE" +then + MERGE_DATE=$(date +'%Y%m%d-%H:%M') +fi +printf 'Result of a run of %s at %s\r\n' \ + "$(basename $0)" "$MERGE_DATE" >temp_file +printf 'Package pools and Packages lists were merged.\r\n' >>temp_file +printf 'The other files stem from the first input ISO.\r\n' >>temp_file +printf '\r\n' >>temp_file +mount_count=0 +for i in $ISO_LIST +do + mount_count=$(expr $mount_count + 1) + mount_point="${MOUNT_TEMPLATE}$mount_count" + printf 'Input ISO: %s\r\n' "$i" >>temp_file + head -2 "$mount_point"/README.txt >>temp_file + printf '\r\n' >>temp_file +done +printf '%s%s\r\n' " --------------------------------------" \ + "----------------------------------------" >>temp_file +printf '\r\n' >>temp_file +cat "$mount_point_1"/README.txt >>temp_file + +mv temp_file merged_README.txt +echo "Done." >&2 + + +## Merge package description files +echo >&2 +echo "Merging package description files ..." >&2 + +# /md5sum.txt seems to be the only overall package list +for i in $MOUNT_LIST +do + cat "$i"/md5sum.txt +done | sort >merged_md5sum.txt + +# Determine the files which are mentioned with checksum in main Release files +path_list=$(for i in $MOUNT_LIST + do + extract_checksum_paths "$i"/dists/"$dist"/Release + done | awk '{print $3}' | sort | uniq ) + +# Merge .gz files (Release should not be merged. Unclear what others need.) +for i in $path_list +do + if echo "$i" | grep -v '.gz$' >/dev/null + then +continue + fi + + echo "Merging: merged_dists/${dist}/$i" >&2 + + # make missing directories in merged_dists/"$dist"/ + if test -e "$(dirname merged_dists/"$dist"/"$i")" + then + dummy=dummy + else + if mkdir -p "$(dirname merged_dists/"$dist"/"$i")" + then + dummy=dummy + else + echo "--- Cannot create directory $(dirname merged_dists/"$dist"/"$i")" >&2 + cleanup_and_end 3 + fi + fi + + test -e temp_file && rm temp_file + for mount_point in $MOUNT_LIST + do + if test -e "$mount_point"/dists/"$dist"/"$i" + then + if test -e temp_file + then + if test -n "$(tail -1 temp_file)" + then + echo >>temp_file + fi + fi + gunzip <"$mount_point"/dists/"$dist"/"$i" >>temp_file + fi + done + if test -e temp_file + then + gzip merged_dists/"$dist"/"$i" + rm temp_file + fi + +done + + +## Update dists/"$dist"/Release +echo >&2 +echo "Updating dists/${dist}/Release ..." >&2 + +extract_release_head merged_dists/"$dist"/Release >temp_file + +# Re-create "MD5Sum:", "SHA1:", "SHA256:", "SHA512:" sections +for cmd in md5sum sha1sum sha256sum sha512sum +do + if type "$cmd" >/dev/null + then + case "$cmd" in + md5sum) echo "MD5Sum:" ;; + sha1sum) echo "SHA1:" ;; + sha256sum) echo "SHA256:" ;; + sha512sum) echo "SHA512:" ;; + esac + for i in $path_list + do + file=merged_dists/"$dist"/"$i" + if test -e "$file" + then + sum=$("$cmd" "$file" | awk '{print $1}') + size=$(stat -c '%s' "$file") + elif test -e "$file".gz + then + sum=$(gunzip <"$file".gz | "$cmd" | awk '{print $1}') + size=$(gunzip <"$file".gz | wc -c) + else + continue + fi + list_path=$(echo "$file" | sed -e 's/^merged_dists\/'"$dist"'\///') + printf ' %s %8ld %s\n' "$sum" "$size" "$list_path" + done + fi +done >>temp_file + +mv temp_file merged_dists/"$dist"/Release +echo "Done." >&2 + + +## Produce the new ISO image +echo >&2 +echo "Producing result ISO image ..." >&2 + +# Create file with list of pool -map commands for all but the first ISO +for mount_point in $MOUNT_LIST +do + if test "$mount_point" = "$mount_point_1" + then + dummy=dummy + echo "Planned as imported package pool : ${mount_point}/pool" >&2 + else + echo "Planned for merging into package pool: ${mount_point}/pool" >&2 + echo " -map ${mount_point}/pool /pool" >>temp_file + fi +done + +if test -z "$XORRISO" +then + XORRISO=xorriso +fi + +x=$(type -p "$XORRISO") +if test -z "$x" +then + echo "Running as xorriso program: $XORRISO" >&2 +else + echo "Running as xorriso program: $x" >&2 +fi +echo >&2 + +if "$XORRISO" \ + -no_rc \ + -indev "$iso_1" \ + -outdev "$RESULT_ISO" \ + -options_from_file temp_file \ + -map merged_dists /dists \ + -map merged_md5sum.txt /md5sum.txt \ + -map merged_README.txt /README.txt \ + -chown_r 0 /dists /md5sum.txt /README.txt -- \ + -chgrp_r 0 /dists /md5sum.txt /README.txt -- \ + -chmod_r a-w /dists /md5sum.txt -- \ + -chmod_r a=r /README.txt -- \ + -boot_image any replay \ + -stdio_sync off \ + -padding included \ + -compliance no_emul_toc +then + echo "Run of xorriso program completed without error indication." >&2 +else + echo "--- Run of xorriso program ends with problem indication." >&2 + cleanup_and_end 4 +fi + + +## Finish + +cleanup_and_end 0 + diff --git a/test/merge_debian_isos.sig b/test/merge_debian_isos.sig new file mode 100644 index 0000000000000000000000000000000000000000..333a7fa10f6308d2cacbf5a780ad52ae21a268bd GIT binary patch literal 72 zcmV-O0Jr~$Mg#y60ssaD0%EDDN&pH85b4X`z^lNhRA<-#pJpJHD`9IkpOU%hzr2{m el!mD)YXF@YancD+A^T`(PqKFWblm&8F2g(TQXkg< literal 0 HcmV?d00001