Added basic Gearman worker/client

Client is only for testing purposes, real client will be part of WebUI
Worker needs better error handling, does work in current form though
Added progress reporting hack in create_image.sh, going to replace this with a better system soon

1 files changed, 50 insertions, 50 deletions

diff --git a/create_image.sh b/create_image.sh
index 03466b1..3cee693 100755
--- a/create_image.sh
+++ b/create_image.sh
@@ -14,8 +14,11 @@ hash qemu-img 2>&- || { echo >&2 "Gentoaster requires qemu-img, but it's not ins
 hash extlinux 2>&- || { echo >&2 "Gentoaster requires extlinux, but it's not installed."; exit 1; }
 hash sfdisk 2>&- || { echo >&2 "Gentoaster requires sfdisk, but it's not installed."; exit 1; }
 
+# Figure out where we are
+RUNNING_DIRECTORY=$(cd `dirname $0` && pwd)
+
 # Load shflags to parse flags
-. ./lib/shflags
+. ${RUNNING_DIRECTORY}/lib/shflags
 
 DEFINE_string 'config' '' 'configuration to build from' 
 DEFINE_string 'proxy' '' 'HTTP proxy to use for emerges'
@@ -30,7 +33,7 @@ if [ -z ${FLAGS_config} ]; then
 fi
 
 # Parse the configuration we took as an arg
-source parse_config.sh ${FLAGS_config}
+source ${RUNNING_DIRECTORY}/parse_config.sh ${FLAGS_config}
 
 # Generate a few helper variables using the configuration file
 
@@ -40,7 +43,7 @@ IMAGE_BYTES=$(( ${IMAGE_MEGABYTES} * 1024 * 1024 ))
 IMAGES_OUTPUT_PATH=`pwd`
 IMAGE_WORK_PATH="${IMAGES_OUTPUT_PATH}/${BUILD_ID}"
 LOG_FILE="${IMAGE_WORK_PATH}/log.txt"
-TOOL_RES_PATH=`pwd`/res
+TOOL_RES_PATH=${RUNNING_DIRECTORY}/res
 
 NUM_JOBS=$(( `grep -c processor /proc/cpuinfo`+1 ))
 STAGE3_URL="http://distribution.hexxeh.net/gentoo/stage3-i686-latest.tar.bz2"
@@ -48,34 +51,33 @@ PORTAGE_URL="http://distribution.hexxeh.net/gentoo/portage-latest.tar.bz2"
 BINHOST_URL="http://tinderbox.dev.gentoo.org/default-linux/x86"
 EMERGE_PROXY="${FLAGS_proxy}"
 
-echo "Creating build working directory"
+echo "Step 1: Creating build working directory"
 mkdir -p ${IMAGE_WORK_PATH}
 cd ${IMAGE_WORK_PATH}
-
-echo "Starting log at ${LOG_FILE}..."
 echo "" > ${LOG_FILE}
 
 # Create disk iamge
 BYTES_PER_CYLINDER=$(( 512*63*255 ))
 CYLINDERS=$(( ${IMAGE_BYTES}/${BYTES_PER_CYLINDER} ))
 REAL_IMAGE_BYTES=$(( (${CYLINDERS}+1)*${BYTES_PER_CYLINDER} ))
-echo "Creating image ${IMAGE_NAME}, size ${REAL_IMAGE_BYTES} bytes"
+echo "Step 2: Creating image ${IMAGE_NAME}, size ${REAL_IMAGE_BYTES} bytes"
 qemu-img create -f raw ${IMAGE_NAME} ${REAL_IMAGE_BYTES} &>> ${LOG_FILE}
 
 # Clean up old mounts
 cleanup_mounts() {
-	echo "Cleaning up old mounts"
-	umount -d ${IMAGE_WORK_PATH}/rootfs/dev/pts &>> ${LOG_FILE}
-	umount -d ${IMAGE_WORK_PATH}/rootfs/dev/shm &>> ${LOG_FILE}
-	umount -d ${IMAGE_WORK_PATH}/rootfs/dev &>> ${LOG_FILE}
-	umount -d ${IMAGE_WORK_PATH}/rootfs/proc &>> ${LOG_FILE}
-	umount -d ${IMAGE_WORK_PATH}/rootfs/boot &>> ${LOG_FILE}
-	umount -d ${IMAGE_WORK_PATH}/rootfs &>> ${LOG_FILE}
+	sleep 2
+	umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev/pts &>> ${LOG_FILE}
+	umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev/shm &>> ${LOG_FILE}
+	umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev &>> ${LOG_FILE}
+	umount -d -f ${IMAGE_WORK_PATH}/rootfs/proc &>> ${LOG_FILE}
+	umount -d -f ${IMAGE_WORK_PATH}/rootfs/boot &>> ${LOG_FILE}
+	umount -d -f ${IMAGE_WORK_PATH}/rootfs &>> ${LOG_FILE}
+	sleep 2
 }
 cleanup_mounts
 
 # Create partition table
-echo "Writing partition table"
+echo "Step 3: Writing partition table"
 echo -e "\x55\xaa" | dd bs=1 count=2 seek=510 of=${IMAGE_NAME} conv=notrunc &>> ${LOG_FILE}
 LOOP_DEV_IMAGE=`losetup -f`
 losetup --sizelimit ${REAL_IMAGE_BYTES} ${LOOP_DEV_IMAGE} ${IMAGE_NAME} &>> ${LOG_FILE}
@@ -89,7 +91,7 @@ sleep 2
 losetup -d ${LOOP_DEV_IMAGE}
 
 # Mounting new root and boot
-echo "Creating filesystems"
+echo "Step 4: Creating filesystems"
 
 LOOP_DEV_BOOT=`losetup -f`
 losetup -o $(( 512 * 2048 )) --sizelimit $(( ${BOOT_MEGABYTES} * 1024 * 1024 )) ${LOOP_DEV_BOOT} ${IMAGE_NAME}
@@ -109,7 +111,7 @@ mkswap ${LOOP_DEV_SWAP} &>> ${LOG_FILE}
 sleep 2 
 losetup -d ${LOOP_DEV_SWAP}
 
-echo "Mounting fileystems"
+echo "Step 5: Mounting fileystems"
 
 mkdir -p rootfs
 mount -o loop,offset=$(( ( 512 * 2048 )  + ( ${BOOT_MEGABYTES} * 1024 * 1024 ) + ( ${SWAP_MEGABYTES} * 1024 * 1024 ) )) ${IMAGE_NAME} rootfs
@@ -125,7 +127,7 @@ then
 	wget ${STAGE3_URL} -O ${TOOL_RES_PATH}/stage3.tar.bz2 &>> ${LOG_FILE}
 fi
 
-echo "Extracting Stage 3"
+echo "Step 6: Extracting Stage 3"
 tar jxf ${TOOL_RES_PATH}/stage3.tar.bz2 &>> ${LOG_FILE}
 
 if [ ! -f ${TOOL_RES_PATH}/portage-latest.tar.bz2 ];
@@ -134,17 +136,17 @@ then
 	wget ${PORTAGE_URL} -O ${TOOL_RES_PATH}/portage-latest.tar.bz2 &>> ${LOG_FILE}
 fi
 
-echo "Extracting Portage snapshot"
+echo "Step 7: Extracting Portage snapshot"
 cd usr
 tar jxf ${TOOL_RES_PATH}/portage-latest.tar.bz2 &>> ${LOG_FILE}
 
-echo "Setting up chroot"
+echo "Step 8: Setting up chroot"
 cd ..
 mount -t proc /proc proc
 mount --rbind /dev dev
 cp -L /etc/resolv.conf etc/resolv.conf
 
-echo "Setting up make.conf"
+echo "Step 9: Setting up make.conf"
 mkdir -p usr/portage/packages
 echo "PORTAGE_BINHOST=\"${BINHOST_URL}\"" >> etc/make.conf
 echo "PKGDIR=\"/usr/portage/packages\"" >> etc/make.conf
@@ -158,67 +160,66 @@ if [[ ${OUTPUT_FORMAT} = "vbox" ]]; then
 fi
 
 if [ -n $EMERGE_PROXY ]; then
-	echo "Enabling HTTP proxy"
+	echo "Enabling HTTP proxy" &>> ${LOG_FILE}
 	echo "http_proxy=\"${EMERGE_PROXY}\"" >> etc/make.conf
 fi
 
-echo "Setting up package.use"
+echo "Step 10: Setting up package.use"
 mkdir -p etc/portage
 echo ${PACKAGE_USE} >> etc/portage/package.use
 
-echo "Setting up package.accept_keywords"
+echo "Step 11: Setting up package.accept_keywords"
 echo ${PACKAGE_ACCEPT_KEYWORDS} >> etc/portage/package.accept_keywords
 
-echo "Entering chroot"
+echo "Step 12: Chroot setup"
 linux32 chroot . env-update &>> ${LOG_FILE}
-
-echo "Loading profile"
 linux32 chroot . /bin/bash /etc/profile &>> ${LOG_FILE}
 
-echo "Setting timezone to ${TIMEZONE}"
+echo "Step 13: Setting timezone to ${TIMEZONE}"
 linux32 chroot . cp /usr/share/zoneinfo/${TIMEZONE} /etc/localtime &>> ${LOG_FILE}
 
-echo "Setting hostname to ${HOSTNAME}"
+echo "Step 14: Setting hostname to ${HOSTNAME}"
 linux32 chroot . /bin/bash -c "echo 127.0.0.1 ${HOSTNAME}.local ${HOSTNAME} localhost > /etc/hosts" &>> ${LOG_FILE}
 
-echo "Copying new fstab"
+echo "Step 15: Copying new fstab"
 cp ${TOOL_RES_PATH}/fstab etc/fstab &>> ${LOG_FILE}
 
-echo "Setting up networking"
+echo "Step 16: Setting up networking"
 echo 'config_eth0=( "dhcp" )' > etc/conf.d/net
 cp etc/init.d/net.lo etc/init.d/net.eth0
 linux32 chroot . rc-update add net.eth0 default &>> ${LOG_FILE}
 
+echo "Step 17: Setting up kernel"
 # If we got the flag, used a cached kernel to reduce build times for testing
-if [[ ${FLAGS_cachedkernel} = ${FLAGS_true} ]]; then
-	echo "Using cached kernel"
+if [[ ${FLAGS_cachedkernel} -eq ${FLAGS_TRUE} ]]; then
+	echo "Using cached kernel" &>> ${LOG_FILE}
 	cp ${TOOL_RES_PATH}/bzImage boot/kernel
 	cp -R ${TOOL_RES_PATH}/kernelmodules/* lib/modules/
 else
-	echo "Downloading/installing kernel sources"
+	echo "Downloading/installing kernel sources" &>> ${LOG_FILE}
 	linux32 chroot . emerge gentoo-sources &>> ${LOG_FILE}
 
-	echo "Copying kernel configuration"
+	echo "Copying kernel configuration" &>> ${LOG_FILE}
 	cp ${TOOL_RES_PATH}/kernelconfig usr/src/linux/.config
 
-	echo "Building kernel"
+	echo "Building kernel" &>> ${LOG_FILE}
 	linux32 chroot . make -C /usr/src/linux -j${NUM_JOBS} &>> ${LOG_FILE}
 
-	echo "Installing kernel"
+	echo "Installing kernel" &>> ${LOG_FILE}
 	linux32 chroot . make -C /usr/src/linux modules_install &>> ${LOG_FILE}
 	usr/src/linux/arch/i386/boot/bzImage boot/kernel &>> ${LOG_FILE}
 fi
 
-echo "Setting root password"
+echo "Step 18: Setting root password"
 linux32 chroot . /bin/bash -c "echo 'root:${ROOT_PASSWORD}' | chpasswd" &>> ${LOG_FILE}
 
-echo "Processing packages list"
+echo "Step 19: Processing packages list"
 for PACKAGE in ${PACKAGES_LIST}; do
-	echo "Installing ${PACKAGE}"
+	echo "Installing ${PACKAGE}" &>> ${LOG_FILE}
 	linux32 chroot . emerge --jobs=${NUM_JOBS} ${PACKAGE} &>> ${LOG_FILE}
 done
 
-echo "Adding default user"
+echo "Step 20: Adding default user"
 linux32 chroot . useradd -g users -G lp,wheel,audio,cdrom,portage -m ${DEFAULT_USERNAME}
 linux32 chroot . /bin/bash -c "echo '${DEFAULT_USERNAME}:${DEFAULT_PASSWORD}' | chpasswd" &>> ${LOG_FILE}
 
@@ -231,32 +232,31 @@ then
 	linux32 chroot . usermod -a vboxguest ${DEFAULT_USERNAME}
 fi
 
-echo "Cleaning up make.conf"
+echo "Step 21: Cleaning up make.conf"
 if [ -n ${EMERGE_PROXY} ]; then
 	sed -i '/http_proxy/ d' etc/make.conf
 fi
 sed -i '/MAKEOPTS/ d' etc/make.conf
 
-echo "Installing extlinux"
-extlinux --heads 255 --sectors 63 --install boot
-dd if=/usr/lib/extlinux/mbr.bin of=../${IMAGE_NAME} conv=notrunc
+echo "Step 22: Installing extlinux"
+extlinux --heads 255 --sectors 63 --install boot &>> ${LOG_FILE}
+dd if=/usr/lib/extlinux/mbr.bin of=../${IMAGE_NAME} conv=notrunc &>> ${LOG_FILE}
 cp ${TOOL_RES_PATH}/extlinux.conf boot/
 cd  ..
 cleanup_mounts
-
 case "${OUTPUT_FORMAT}" in
 	"raw" )
-		echo "Already in raw format, not converting"
+		echo "Already in raw format, not converting" &>> ${LOG_FILE}
 		IMAGE_OUT="${BUILD_ID}.image"
 		;;
 	"vbox" )
-		echo "Converting image from RAW to VDI"
+		echo "Converting image from RAW to VDI" &>> ${LOG_FILE}
 		qemu-img convert -O vdi ${IMAGE_NAME} ${BUILD_ID}.vdi
 		rm -rf ${IMAGE_NAME}
 		IMAGE_OUT="${BUILD_ID}.vdi"
 		;;
 	"vmware" )
-		echo "Converting image from RAW to VMDK"
+		echo "Converting image from RAW to VMDK" &>> ${LOG_FILE}
 		qemu-img convert -O vmdk ${IMAGE_NAME} ${BUILD_ID}.vmdk
 		rm -rf ${IMAGE_NAME}
 		IMAGE_OUT="${BUILD_ID}.vmdk"
@@ -266,6 +266,6 @@ mv ${IMAGE_OUT} ${IMAGES_OUTPUT_PATH}/${IMAGE_OUT}
 mv ${LOG_FILE} ${IMAGES_OUTPUT_PATH}/${BUILD_ID}.log
 rm -rf ${IMAGE_WORK_PATH}
 
-echo "Image build completed!"
+echo "Step 23: Image build completed!"
 echo "Your image is here: ${IMAGES_OUTPUT_PATH}/${IMAGE_OUT}"
 echo "Your log file is here: ${IMAGES_OUTPUT_PATH}/${BUILD_ID}.log"