# HG changeset patch
# User Rob Landley <rob@landley.net>
# Date 1251272277 18000
# Node ID 8129df56091b4261fa5c0547c37afc857d420d58
# Parent  5dd054d1c7894c0a1e01477981444ea2ab589df4
Extended setupfor/cleanup to create binary package tarballs if the configure option BINARY_PACKAGE_TARBALLS is set.  These tarballs extract into the current directory and add all the changed files installed by this package build, so you can pick and choose when assembling your own filesystem.

diff -r 5dd054d1c789 -r 8129df56091b config
--- a/config	Mon Aug 24 00:29:35 2009 -0500
+++ b/config	Wed Aug 26 02:37:57 2009 -0500
@@ -123,3 +123,8 @@
 # Don't update the title bar in the display
 
 # export NO_TITLE_BAR=1
+
+# Create a tarball with the files installed by each individual package at
+# each stage of the build.
+
+# export BINARY_PACKAGE_TARBALLS=1
diff -r 5dd054d1c789 -r 8129df56091b cross-compiler.sh
--- a/cross-compiler.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/cross-compiler.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -22,7 +22,7 @@
 
 # Build and install binutils
 
-setupfor binutils build-binutils &&
+setupfor binutils build-binutils
 AR=ar AS=as LD=ld NM=nm OBJDUMP=objdump OBJCOPY=objcopy \
 	"${CURSRC}/configure" --prefix="${STAGE_DIR}" --host=${CROSS_HOST} \
 	--target=${CROSS_TARGET} --with-lib-path=lib --disable-nls \
@@ -31,11 +31,10 @@
 make -j $CPUS configure-host &&
 make -j $CPUS CFLAGS="-O2 $STATIC_FLAGS" &&
 make -j $CPUS install &&
-cd .. &&
 mkdir -p "${STAGE_DIR}/include" &&
-cp binutils/include/libiberty.h "${STAGE_DIR}/include"
+cp "$CURSRC/include/libiberty.h" "${STAGE_DIR}/include"
 
-cleanup binutils build-binutils
+cleanup build-binutils
 
 # Build and install gcc
 
@@ -54,9 +53,9 @@
 
 make -j $CPUS all-gcc LDFLAGS="$STATIC_FLAGS" &&
 make -j $CPUS install-gcc &&
-cd ..
 
-cleanup gcc-core build-gcc
+# We're done with that source and could theoretically cleanup gcc-core and
+# build-gcc here, but we still need the timestamps if we do binary packaging.
 
 echo Fixup toolchain... &&
 
@@ -82,9 +81,13 @@
 rm "${ARCH}-c++" &&
 ln -s "${ARCH}-g++" "${ARCH}-rawc++" &&
 ln -s "${ARCH}-gcc" "${ARCH}-g++" &&
-ln -s "${ARCH}-gcc" "${ARCH}-c++"
+ln -s "${ARCH}-gcc" "${ARCH}-c++" &&
 
-cleanup "${STAGE_DIR}"/{lib/gcc,{libexec/gcc,gcc/lib}/install-tools}
+# When tarring up individual binary packages, we want this one to be called
+# "gcc" and that's not what we fed to setupfor, so rename it.
+
+mv "$WORK"/gcc-core "$WORK"/gcc
+PACKAGE=gcc cleanup build-gcc "${STAGE_DIR}"/{lib/gcc,{libexec/gcc,gcc/lib}/install-tools}
 
 # Install kernel headers.
 
@@ -92,10 +95,9 @@
 # Install Linux kernel headers (for use by uClibc).
 make -j $CPUS headers_install ARCH="${KARCH}" INSTALL_HDR_PATH="${STAGE_DIR}" &&
 # This makes some very old package builds happy.
-ln -s ../sys/user.h "${STAGE_DIR}/include/asm/page.h" &&
-cd ..
+ln -s ../sys/user.h "${STAGE_DIR}/include/asm/page.h"
 
-cleanup linux
+cleanup
 
 # Build and install uClibc
 
@@ -108,9 +110,8 @@
 do
   cp utils/"$i".host "$STAGE_DIR/bin/$ARCH-$i" || dienow
 done
-cd ..
 
-cleanup uClibc
+cleanup
 
 cat > "${STAGE_DIR}"/README << EOF &&
 Cross compiler for $ARCH
diff -r 5dd054d1c789 -r 8129df56091b host-tools.sh
--- a/host-tools.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/host-tools.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -31,6 +31,9 @@
 
 STAGE_DIR="${HOSTTOOLS}"
 
+# Blank $WORK but accept $STAGE_DIR if it exists.  Re-running this script
+# should be a NOP.
+
 blank_tempdir "${WORK}"
 mkdir -p "${STAGE_DIR}" || dienow
 
@@ -111,9 +114,8 @@
       [ ! -f "${STAGE_DIR}/$i" ] &&
         (ln -sf busybox "${STAGE_DIR}/$i" || dienow)
     done
-    cd ..
 
-    cleanup busybox
+    cleanup
   fi
 
   # Build toybox
@@ -131,9 +133,8 @@
     else
       make install_flat PREFIX="${STAGE_DIR}" || dienow
     fi
-    cd ..
 
-    cleanup toybox
+    cleanup
   fi
 
   # Create symlinks to the host toolchain.  We need a usable existing host
@@ -186,10 +187,9 @@
   setupfor distcc &&
   ./configure --with-included-popt --disable-Werror &&
   make -j "$CPUS" &&
-  cp distcc distccd "${STAGE_DIR}" &&
-  cd ..
+  cp distcc distccd "${STAGE_DIR}"
 
-  cleanup distcc
+  cleanup
 fi
 
 # Build genext2fs.  We use it to build the ext2 image to boot qemu with
@@ -200,10 +200,9 @@
   setupfor genext2fs &&
   ./configure &&
   make -j $CPUS &&
-  cp genext2fs "${STAGE_DIR}" &&
-  cd ..
+  cp genext2fs "${STAGE_DIR}"
 
-  cleanup genext2fs
+  cleanup
 fi
 
 # Build e2fsprogs.
@@ -226,10 +225,9 @@
   ./configure --disable-tls --enable-htree &&
   make -j "$CPUS" &&
   cp misc/{mke2fs,tune2fs} resize/resize2fs "${STAGE_DIR}" &&
-  cp e2fsck/e2fsck "$STAGE_DIR"/fsck.ext2 &&
-  cd ..
+  cp e2fsck/e2fsck "$STAGE_DIR"/fsck.ext2
 
-  cleanup e2fsprogs
+  cleanup
 fi
 
 # Squashfs is an alternate packaging option.
@@ -239,10 +237,9 @@
   setupfor squashfs &&
   cd squashfs-tools &&
   make -j $CPUS &&
-  cp mksquashfs unsquashfs "${STAGE_DIR}" &&
-  cd ..
+  cp mksquashfs unsquashfs "${STAGE_DIR}"
 
-  cleanup squashfs
+  cleanup
 fi
 
 # Here's some stuff that isn't used to build a cross compiler or system
@@ -272,10 +269,9 @@
     make -j $CPUS &&
     # Copy the executable files and ROM files
     cp $(find -type f -perm +111 -name "qemu*") "$STAGE_DIR" &&
-    cp -r pc-bios "$STAGE_DIR" &&
-    cd ..
+    cp -r pc-bios "$STAGE_DIR"
 
-    cleanup qemu
+    cleanup
   else
     # Symlink qemu out of the host, if found.  Since run-from-build.sh uses
     # $PATH=.../build/host if it exists, add the various qemu instances to that.
diff -r 5dd054d1c789 -r 8129df56091b root-filesystem.sh
--- a/root-filesystem.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/root-filesystem.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -28,8 +28,8 @@
 echo -e "$NATIVE_COLOR"
 echo "=== Building $STAGE_NAME"
 
+blank_tempdir "$STAGE_DIR"
 blank_tempdir "$WORK"
-blank_tempdir "$STAGE_DIR"
 
 # Determine which directory layout we're using
 
@@ -53,10 +53,9 @@
 # Install Linux kernel headers (for use by uClibc).
 make headers_install -j "$CPUS" ARCH="${KARCH}" INSTALL_HDR_PATH="$ROOT_TOPDIR" &&
 # This makes some very old package builds happy.
-ln -s ../sys/user.h "$ROOT_TOPDIR/include/asm/page.h" &&
-cd ..
+ln -s ../sys/user.h "$ROOT_TOPDIR/include/asm/page.h"
 
-cleanup linux
+cleanup
 
 # Build and install uClibc.  (We could just copy the one from the compiler
 # toolchain, but this is cleaner.)
@@ -85,9 +84,7 @@
   done
 fi
 
-cd ..
-
-cleanup uClibc
+cleanup
 
 if [ "$NATIVE_TOOLCHAIN" == "none" ]
 then
@@ -117,11 +114,10 @@
 make -j $CPUS configure-host &&
 make -j $CPUS CFLAGS="-O2 $STATIC_FLAGS" &&
 make -j $CPUS install &&
-cd .. &&
 mkdir -p "$ROOT_TOPDIR/include" &&
-cp binutils/include/libiberty.h "$ROOT_TOPDIR/include"
+cp "$CURSRC/include/libiberty.h" "$ROOT_TOPDIR/include"
 
-cleanup binutils build-binutils
+cleanup build-binutils
 
 # Build and install native gcc, with c++ support this time.
 
@@ -166,9 +162,9 @@
 cd "$CROSS_TARGET"/libstdc++-v3/libsupc++ &&
 make -j $CPUS &&
 mv .libs/libsupc++.a "$ROOT_TOPDIR"/lib &&
-cd ../../../..
 
-cleanup gcc-core build-gcc
+# We're done with that source and could theoretically cleanup gcc-core and
+# build-gcc here, but we still need the timestamps if we do binary packaging.
 
 # Move the gcc internal libraries and headers somewhere sane
 
@@ -190,9 +186,14 @@
 mv "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}g++" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}rawg++" &&
 ln "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}gcc" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}g++" &&
 rm "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}c++" &&
-ln -s "${PROGRAM_PREFIX}g++" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}c++"
+ln -s "${PROGRAM_PREFIX}g++" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}c++" &&
 
-cleanup "$ROOT_TOPDIR"/{lib/gcc,gcc/lib/install-tools,bin/${ARCH}-unknown-*}
+# When tarring up individual binary packages, we want this one to be called
+# "gcc" and that's not what we fed to setupfor, so rename it.
+
+mv "$WORK"/{gcc-core,gcc}
+PACKAGE=gcc cleanup build-gcc \
+  "$ROOT_TOPDIR"/{lib/gcc,gcc/lib/install-tools,bin/${ARCH}-unknown-*}
 
 # Tell future packages to link against the libraries in the new root filesystem,
 # rather than the ones in the cross compiler directory.
@@ -214,10 +215,9 @@
 mv "$ROOT_TOPDIR"/c++/lib/* "$ROOT_TOPDIR"/lib &&
 rm -rf "$ROOT_TOPDIR"/c++/{lib,bin} &&
 ln -s libuClibc++.so "$ROOT_TOPDIR"/lib/libstdc++.so &&
-ln -s libuClibc++.a "$ROOT_TOPDIR"/lib/libstdc++.a &&
-cd ..
+ln -s libuClibc++.a "$ROOT_TOPDIR"/lib/libstdc++.a
 
-cleanup uClibc++
+cleanup
 
 fi # End of NATIVE_TOOLCHAIN build
 
@@ -240,15 +240,13 @@
   cp toybox "$ROOT_TOPDIR/bin" &&
   ln -s toybox "$ROOT_TOPDIR/bin/patch" &&
   ln -s toybox "$ROOT_TOPDIR/bin/oneit" &&
-  ln -s toybox "$ROOT_TOPDIR/bin/netcat" &&
-  cd ..
+  ln -s toybox "$ROOT_TOPDIR/bin/netcat"
 else
   CFLAGS="$CFLAGS $STATIC_FLAGS" \
-    make install_flat PREFIX="$ROOT_TOPDIR"/bin CROSS="${ARCH}-" &&
-  cd ..
+    make install_flat PREFIX="$ROOT_TOPDIR"/bin CROSS="${ARCH}-"
 fi
 
-cleanup toybox
+cleanup
 
 # Build and install busybox
 
@@ -258,18 +256,15 @@
 LDFLAGS="$LDFLAGS $STATIC_FLAGS" \
   make -j $CPUS CROSS_COMPILE="${ARCH}-" $VERBOSITY &&
 make busybox.links &&
-cp busybox "$ROOT_TOPDIR/bin"
-
-[ $? -ne 0 ] && dienow
+cp busybox "$ROOT_TOPDIR/bin" || dienow
 
 for i in $(sed 's@.*/@@' busybox.links)
 do
   # Allowed to fail.
-  ln -s busybox "$ROOT_TOPDIR/bin/$i" 2>/dev/null
+  ln -s busybox "$ROOT_TOPDIR/bin/$i" 2>/dev/null || true
 done
-cd ..
 
-cleanup busybox
+cleanup
 
 # Build and install make
 
@@ -277,10 +272,9 @@
 LDFLAGS="$STATIC_FLAGS $LDFLAGS" CC="${ARCH}-cc" ./configure \
   --prefix="$ROOT_TOPDIR" --build="${CROSS_HOST}" --host="${CROSS_TARGET}" &&
 make -j $CPUS &&
-make -j $CPUS install &&
-cd ..
+make -j $CPUS install
 
-cleanup make
+cleanup
 
 # Build and install bash.  (Yes, this is an old version.  It's intentional.)
 
@@ -302,10 +296,9 @@
 make &&
 make install &&
 # Make bash the default shell.
-ln -sf bash "$ROOT_TOPDIR/bin/sh" &&
-cd ..
+ln -sf bash "$ROOT_TOPDIR/bin/sh"
 
-cleanup bash
+cleanup
 
 setupfor distcc
 rsync_cv_HAVE_C99_VSNPRINTF=yes \
@@ -320,9 +313,8 @@
 do
   ln -s ../bin/distcc "$ROOT_TOPDIR/distcc/$i" || dienow
 done
-cd ..
 
-cleanup distcc
+cleanup
 
 # Put statically and dynamically linked hello world programs on there for
 # test purposes.
diff -r 5dd054d1c789 -r 8129df56091b sources/functions.sh
--- a/sources/functions.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/sources/functions.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -97,22 +97,58 @@
   dienow
 }
 
+# Find all files in $STAGE_DIR newer than $CURSRC.
+
+function recent_binary_files()
+{
+  PREVIOUS=
+  (cd "$STAGE_DIR" || dienow
+   # Note $WORK/$PACKAGE != $CURSRC here for renamed packages like gcc-core.
+   find . -newer "$WORK/$PACKAGE/FWL-TIMESTAMP" -depth \
+     | sed -e 's/^.//' -e 's/^.//' -e '/^$/d'
+  ) | while read i
+  do
+    TEMP="${PREVIOUS##"$i"/}"
+
+    if [ $[${#PREVIOUS}-${#TEMP}] -ne $[${#i}+1] ]
+    then
+      # Because the expanded $i might have \ chars in it, that's why.
+      echo -n "$i"
+      echo -ne '\0'
+    fi
+    PREVIOUS="$i"
+  done
+}
+
 # Strip the version number off a tarball
 
 function cleanup()
 {
+  # If package build exited with an error, do not continue.
 
   [ $? -ne 0 ] && dienow
 
+  if [ ! -z "$BINARY_PACKAGE_TARBALLS" ]
+  then
+    TARNAME="$PACKAGE-$STAGE_NAME-${ARCH_NAME}".tar.bz2
+    echo -n Creating "$TARNAME"
+    { recent_binary_files | xargs -0 tar -cjvf \
+        "$BUILD/${TARNAME}".tar.bz2 -C "$STAGE_DIR" || dienow
+    } | dotprogress
+  fi
+
   if [ ! -z "$NO_CLEANUP" ]
   then
-    echo "skip cleanup $@"
+    echo "skip cleanup $PACKAGE $@"
     return
   fi
 
-  for i in "$@"
+  # Loop deleting directories
+
+  cd "$WORK" || dienow
+  for i in "$PACKAGE" "$@"
   do
-    unstable "$i" && i="$PACKAGE"
+    [ -z "$i" ] && continue
     echo "cleanup $i"
     rm -rf "$i" || dienow
  done
@@ -198,7 +234,6 @@
   [ "$FILENAME" != "${FILENAME/%\.tar\.bz2/}" ] && DECOMPRESS="j"
   [ "$FILENAME" != "${FILENAME/%\.tar\.gz/}" ] && DECOMPRESS="z"
 
-  cd "${WORK}" &&
   { tar -xv${DECOMPRESS} -f "${SRCDIR}/${FILENAME}" -C "${BUILD}/temp" || dienow
   } | dotprogress
 
@@ -420,6 +455,21 @@
   # Change window title bar to package now
   [ -z "$NO_TITLE_BAR" ] &&
     echo -en "\033]2;$ARCH_NAME $STAGE_NAME $PACKAGE\007"
+
+  # Ugly bug workaround: timestamp granularity in a lot of filesystems is only
+  # 1 second, so find -newer misses things installed in the same second, so we
+  # make sure it's a new second before we start actually doing anything.
+
+  if [ ! -z "$BINARY_PACKAGE_TARBALLS" ]
+  then
+    touch "${CURSRC}/FWL-TIMESTAMP" || dienow
+    TIME=$(date +%s)
+    while true
+    do
+      [ $TIME != "$(date +%s)" ] && break
+      sleep .1
+    done
+  fi
 }
 
 # Figure out what version of a package we last built
diff -r 5dd054d1c789 -r 8129df56091b system-image.sh
--- a/system-image.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/system-image.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -17,8 +17,6 @@
   exit 1
 fi
 
-blank_tempdir "$WORK"
-
 # This little song and dance makes us run in our own session, to prevent the
 # "kill 0" below from taking down the shell that called us when it cleans up
 # our background tasks.  (We run the kernel build and root filesystem image
@@ -43,6 +41,7 @@
 echo "=== Packaging system image from root-filesystem"
 
 blank_tempdir "$STAGE_DIR"
+blank_tempdir "$WORK"
 
 
 [ -z "$SYSIMAGE_TYPE" ] && SYSIMAGE_TYPE=squashfs
@@ -166,10 +165,9 @@
 # Install kernel
 
 [ -d "${TOOLS}/src" ] && cp .config "${TOOLS}"/src/config-linux
-cp "${KERNEL_PATH}" "${STAGE_DIR}/zImage-${ARCH}" &&
-cd ..
+cp "${KERNEL_PATH}" "${STAGE_DIR}/zImage-${ARCH}"
 
-cleanup linux
+cleanup
 
 # Provide qemu's common command line options between architectures.  The lack
 # of ending quotes on -append is intentional, callers append more kernel