Improved speed of md5sum.txt processing by a proposal of Zhang Boyang

test/merge_debian_isos

@@ -26,9 +26,9 @@ usage() {
   echo "Further it creates and finally removes directories mount_template*" >&2
   echo "if they are needed and do not exist when the script starts." >&2
   echo "It depends on the following programs:" >&2
-  echo "  awk, basename, cat, chmod, cp, dirname, expr, fgrep, grep," >&2
+  echo "  awk, basename, cat, chmod, cp, date, dirname, expr, fgrep, grep," >&2
   echo "  gunzip, gzip, head, ls, md5sum, mkdir, mount, mv, rm, rmdir," >&2
-  echo "  sed, sh, sha256sum, sort, stat, sudo, umount, xorriso" >&2
+  echo "  sed, sh, sha256sum, sort, stat, sudo, umount, uniq, xorriso" >&2
   echo "Recommended are: sha1sum, sha512sum" >&2
   echo >&2
   echo "Exported non-empty variable MERGE_DATE enforces a particular" >&2
@@ -239,9 +239,9 @@ fi
 echo >&2
 echo "$(basename "$0") starting with $(expr $# - 2) ISO image files ..." >&2
 
-dep="awk basename cat chmod cp dirname expr fgrep grep"
+dep="awk basename cat chmod cp date dirname expr fgrep grep"
 dep="$dep gunzip gzip head ls md5sum mkdir mount mv rm rmdir sed"
-dep="$dep sha256sum sort stat sudo umount"
+dep="$dep sha256sum sort stat sudo umount uniq"
 missing=0
 for i in $dep
 do
@@ -524,12 +524,6 @@ extract_release_head() {
 # Decide whether to re-compute MD5 and whether to put out the md5sum line
 # Parameters: prev_path prev_md5
 check_and_put_out_md5_grep() {
-
-  if echo "$prev_path" | grep '^\.\/pool\/' >/dev/null 2>&1
-  then
-    echo "$prev_md5  $prev_path"
-    return 0
-  fi
   if echo "$prev_path" | grep '^\.\/firmware\/' >/dev/null 2>&1
   then
     if test "$was_multiple" = 1 && test "$multi_md5_differs" = 1
@@ -567,15 +561,14 @@ check_and_put_out_md5_grep() {
   return 0
 }
 
-# Put out merged_md5sum.txt while removing duplicates
-# and recomputing MD5 if needed
+# Filter lines from md5sum.txt by removing duplicates and recomputing
+# the MD5 if needed and possible.
 polish_md5sum_txt() {
   prev_path=
   prev_md5=
   was_multiple=0
   multi_md5_differs=0
 
-  cat merged_md5sum.txt | \
   while read md5 path
   do
     if test "$path" = "$prev_path"
@@ -608,6 +601,7 @@ polish_md5sum_txt() {
   return 0
 }
 
+
 ## Determine which Debian release is on iso1
 echo >&2
 echo "Determining Debian release in first ISO ..." >&2
@@ -815,7 +809,11 @@ do
   cat "$i"/md5sum.txt
 done | sort -k 2 >merged_md5sum.txt
 
-polish_md5sum_txt >temp_file
+# ./pool files are surely unchanged. Others need some more examination.
+( fgrep ' ./pool/' <merged_md5sum.txt | uniq 
+  fgrep -v ' ./pool/' <merged_md5sum.txt | polish_md5sum_txt ) \
+| sort -k 2 >temp_file
+
 mv temp_file merged_md5sum.txt
 echo "Done." >&2