libburnia
/
libisoburn
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved speed of md5sum.txt processing by a proposal of Zhang Boyang

This commit is contained in:
Thomas Schmitt 2022-06-26 10:58:16 +02:00
parent 072c4e5ab9
commit 0e8227e76a
2 changed files with 12 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
test/merge_debian_isos
View File

@ -26,9 +26,9 @@ usage() {
echo "Further it creates and finally removes directories mount_template*" >&2 echo "Further it creates and finally removes directories mount_template*" >&2
echo "if they are needed and do not exist when the script starts." >&2 echo "if they are needed and do not exist when the script starts." >&2
echo "It depends on the following programs:" >&2 echo "It depends on the following programs:" >&2
echo " awk, basename, cat, chmod, cp, dirname, expr, fgrep, grep," >&2 echo " awk, basename, cat, chmod, cp, date, dirname, expr, fgrep, grep," >&2
echo " gunzip, gzip, head, ls, md5sum, mkdir, mount, mv, rm, rmdir," >&2 echo " gunzip, gzip, head, ls, md5sum, mkdir, mount, mv, rm, rmdir," >&2
echo " sed, sh, sha256sum, sort, stat, sudo, umount, xorriso" >&2 echo " sed, sh, sha256sum, sort, stat, sudo, umount, uniq, xorriso" >&2
echo "Recommended are: sha1sum, sha512sum" >&2 echo "Recommended are: sha1sum, sha512sum" >&2
echo >&2 echo >&2
echo "Exported non-empty variable MERGE_DATE enforces a particular" >&2 echo "Exported non-empty variable MERGE_DATE enforces a particular" >&2
@ -239,9 +239,9 @@ fi
echo >&2 echo >&2
echo "$(basename "$0") starting with $(expr $# - 2) ISO image files ..." >&2 echo "$(basename "$0") starting with $(expr $# - 2) ISO image files ..." >&2
dep="awk basename cat chmod cp dirname expr fgrep grep" dep="awk basename cat chmod cp date dirname expr fgrep grep"
dep="$dep gunzip gzip head ls md5sum mkdir mount mv rm rmdir sed" dep="$dep gunzip gzip head ls md5sum mkdir mount mv rm rmdir sed"
dep="$dep sha256sum sort stat sudo umount" dep="$dep sha256sum sort stat sudo umount uniq"
missing=0 missing=0
for i in $dep for i in $dep
do do
@ -524,12 +524,6 @@ extract_release_head() {
# Decide whether to re-compute MD5 and whether to put out the md5sum line # Decide whether to re-compute MD5 and whether to put out the md5sum line
# Parameters: prev_path prev_md5 # Parameters: prev_path prev_md5
check_and_put_out_md5_grep() { check_and_put_out_md5_grep() {
if echo "$prev_path" | grep '^\.\/pool\/' >/dev/null 2>&1
then
echo "$prev_md5 $prev_path"
return 0
fi
if echo "$prev_path" | grep '^\.\/firmware\/' >/dev/null 2>&1 if echo "$prev_path" | grep '^\.\/firmware\/' >/dev/null 2>&1
then then
if test "$was_multiple" = 1 && test "$multi_md5_differs" = 1 if test "$was_multiple" = 1 && test "$multi_md5_differs" = 1
@ -567,15 +561,14 @@ check_and_put_out_md5_grep() {
return 0 return 0
} }
# Put out merged_md5sum.txt while removing duplicates # Filter lines from md5sum.txt by removing duplicates and recomputing
# and recomputing MD5 if needed # the MD5 if needed and possible.
polish_md5sum_txt() { polish_md5sum_txt() {
prev_path= prev_path=
prev_md5= prev_md5=
was_multiple=0 was_multiple=0
multi_md5_differs=0 multi_md5_differs=0
cat merged_md5sum.txt | \
while read md5 path while read md5 path
do do
if test "$path" = "$prev_path" if test "$path" = "$prev_path"
@ -608,6 +601,7 @@ polish_md5sum_txt() {
return 0 return 0
} }
## Determine which Debian release is on iso1 ## Determine which Debian release is on iso1
echo >&2 echo >&2
echo "Determining Debian release in first ISO ..." >&2 echo "Determining Debian release in first ISO ..." >&2
@ -815,7 +809,11 @@ do
cat "$i"/md5sum.txt cat "$i"/md5sum.txt
done | sort -k 2 >merged_md5sum.txt done | sort -k 2 >merged_md5sum.txt
polish_md5sum_txt >temp_file # ./pool files are surely unchanged. Others need some more examination.
( fgrep ' ./pool/' <merged_md5sum.txt | uniq
fgrep -v ' ./pool/' <merged_md5sum.txt | polish_md5sum_txt ) \
| sort -k 2 >temp_file
mv temp_file merged_md5sum.txt mv temp_file merged_md5sum.txt
echo "Done." >&2 echo "Done." >&2

BIN
test/merge_debian_isos.sig
View File

Binary file not shown.