Major backend reorganization to split build into steps and allow automatic resuming after failure

15 files changed, 130 insertions, 120 deletions

diff --git a/backend/backend.php b/backend/backend.php
index d5c989e..2874b1a 100755
--- a/backend/backend.php
+++ b/backend/backend.php
@@ -43,56 +43,56 @@ if (posix_geteuid() !== 0)
 	fputs(STDERR, "Not running as root... this is not going to accomplish much.\n");
 if (@file_put_contents($pidfile, posix_getpid()))
 	$unlinkpidfile=true;
-require_once(dirname(__FILE__).'/../shared/include/includes.php'); // USE __DIR__ once 5.3.0 is out (and 2 lines down)
+require_once(dirname(__FILE__).'/../shared/include/includes.php'); // USE __DIR__ once 5.3.0 is out
 require_once(BACKEND.'/include/signals.php');
 declare(ticks=1);
 require_once(SHARED.'/include/dbinit.php');
 while (true) {
 	// TODO check first for builds that need to be resumed (and figure out how to resume things)
-	query('LOCK TABLES `builds` WRITE');
-	$r=query('SELECT * FROM `builds` WHERE `status`=-128 ORDER BY `ctime` ASC LIMIT 1');
-	if ($r->rowCount()) {
-		$build=new sql_build($r->fetch(PDO::FETCH_ASSOC));
+	while (true) {
+		$r=query('SELECT * FROM `builds` WHERE `backend`="'.$S['conf']['backend_id'].'" AND `status` IN ("queued","cancel","uploading","upload_failed","building","got_signal") ORDER BY `ctime` ASC LIMIT 1');
+		if ($r->rowCount())
+			break;
+		else {
+			$r=query('UPDATE `builds` SET `backend`="'.$S['conf']['backend_id'].'" WHERE `status`="queued" AND `backend` IS NULL ORDER BY `ctime` ASC LIMIT 1');
+			if ($r->rowCount() == 0)
+				sleep(5);
+		}
+	}
+	$build=new sql_build($r->fetch(PDO::FETCH_ASSOC));
+	if (!isset($build->start)) {
 		$build->start=time();
-		$build->status=-1;
 		$build->write();
-		query('UNLOCK TABLES');
-		debug('Starting build id='.$build->id);
-		$owner=$build->get_owner();
-		$file=null;
+	}
+	debug('Starting build id='.$build->id);
+	$file=null;
+	$owner=$build->get_owner();
+	$workdir=WORK."/build-$build->id";
+	if (($image=$build->build($workdir)) !== false) {
 		try {
-			if ($S['conf']['split_setup']) {
-				$opt=new sql_buildopt($build->id, 'backend', $S['conf']['backend_id']);
-				$opt->write();
-				unset($opt);
-			}
+			$bundler=$build->get_opt('bundler');
+			$bundle_proc="bundle_$bundler";
+			if (!function_exists($bundle_proc))
+				throw_exception("No bundler function defined for bundler $bundler");
 			$opts=$build->get_opts();
-			$build_proc=$build->module.'_build';
-			require_once(BACKEND."/modules/$build->module/build.php");
-			// TODO check that build_proc exists
-			$workdir=WORK.'/build-'.$build->id;
-			log_status('Creating work directory '.$workdir, mkdir($workdir, 0700));
-			$image=$build_proc($build, $opts, $workdir);
-			require_once(BACKEND."/bundlers/{$opts['bundler']}.php");
-			$proc='bundle_'.$opts['bundler'];
-			$file=$proc($image, $workdir, $opts);
+			$file=$bundle_proc($image, $workdir, $opts);
 			end_internal_task(0); // Just in case
 		} catch (Exception $e) {
+			log_msg('Caught exception: '.$e->getMessage());
 			end_internal_task(1);
-//			log_msg('Caught exception: '.$e->getMessage());
-			$build->status=INGENUE_BUILD_FAILED;
-			xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed.  You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>');
+			$build->status='failed';
+			$build->write();
+			xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed in bundling stage.  You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>');
 		}
 		$build->finish=time();
 		debug('Finished with build id='.$build->id);
 		if (isset($file)) {
 			debug("Completed build successfully");
 			if ($S['conf']['split_setup']) {
-				$build->status=INGENUE_BUILD_UPLOADING;
+				$build->status='uploading';
 				$build->write();
 				$key=randstring(30);
-				$opt=new sql_buildopt($build->id, 'uploadkey', $key);
-				$opt->write();
+				$build->set_opt('uploadkey', $key);
 				$c=curl_init(url('backend/upload_image'));
 				curl_setopt($c, CURLOPT_POST, 1);
 				curl_setopt($c, CURLOPT_POSTFIELDS, array(
@@ -106,30 +106,30 @@ while (true) {
 					debug($result);
 				}
 				if ($result === false || strpos($result, 'Upload successful') === false) {
-					$build->status=INGENUE_BUILD_UPLOAD_FAILED;
+					$build->status='upload_failed';
+					$build->write();
 				} else {
 					debug("Transferred $file... unlinking it");
 					unlink($file);
-					$build->status=INGENUE_BUILD_COMPLETE;
+					$build->status='complete';
+					$build->write();
+					shell_exec('rm -rf '.escapeshellarg($workdir));
 				}
 			} else {
-				$build->status=INGENUE_BUILD_COMPLETE;
+				$build->status='complete';
 				$base=basename($file);
 				$ext=substr($base, strpos($base, '.'));
 				rename($file, COMPLETED.'/build-'.$build->id.$ext);
+				$build->write();
+				shell_exec('rm -rf '.escapeshellarg($workdir));
 			}
 			xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build finished', 'Your build has completed successfully.  You can find more information and download the completed image at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>');
 		}
-		$build->write();
-		if (!$S['conf']['debug']) {
-			shell_exec('rm -rf "'.$workdir.'"');
-		}
-		unset($build);
 	} else {
-		query('UNLOCK TABLES');
+		$build->status='failed';
+		$build->write();
+		xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed.  You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>');
 	}
-	//	log_msg('Sleeping...', false);
-	sleep(5);
-	//	log_msg("done");
+	unset($build);
 }
 ?>
diff --git a/backend/bundlers/cd.inc b/backend/bundlers/cd.inc
new file mode 100644
index 0000000..6fb5aac
--- /dev/null
+++ b/backend/bundlers/cd.inc
@@ -0,0 +1,12 @@
+<?php
+$profile=new sql_gentoo_profile($opts['profile']);
+$headers=$profile->get_headers();
+if (strpos($headers['chost'], 'x86_64') === false)
+	$minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso';
+else
+	$minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso';
+makedir("$I/boot");
+execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'");
+//file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n");
+execute_command('Extract kernel modules from SquashFS to image', "unsquashfs -i -d '$W/modules' '$W/image.squashfs' /lib/modules /lib64/modules; cp -avT '$W/modules' '$I'; rm -rf '$W/modules'");
+?>
diff --git a/backend/bundlers/installcd.php b/backend/bundlers/installcd.php
index 9a4f857..63ffb5a 100644
--- a/backend/bundlers/installcd.php
+++ b/backend/bundlers/installcd.php
@@ -1,17 +1,8 @@
 <?php
 function bundle_installcd($I, $W, &$opts) {
-	$profile=new sql_gentoo_profile($opts['profile']);
-	$headers=$profile->get_headers();
-	if (strpos($headers['chost'], 'x86_64') === false)
-		$minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso';
-	else
-		$minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso';
-	makedir("$I/boot");
-	execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'");
-	file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n");
-	execute_command('Extract kernel modules from SquashFS', "unsquashfs -i -d '$W/modules' -e '$W/unsquashfs-files' '$W/image.squashfs'");
-	execute_command('Copy extracted modules to image', "cp -av '$W/modules/*' '$I/'");
+	require(dirname(__FILE__).'/cd.inc');
 	execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' .");
 	execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' add / '$W/image.tar.bz2' write '$W/image.iso'");
 	return "$W/image.iso";
 }
+?>
diff --git a/backend/bundlers/livecd.php b/backend/bundlers/livecd.php
index d8a40fd..740c72b 100644
--- a/backend/bundlers/livecd.php
+++ b/backend/bundlers/livecd.php
@@ -1,17 +1,8 @@
 <?php
 function bundle_livecd($I, $W, &$opts) {
-	$profile=new sql_gentoo_profile($opts['profile']);
-	$headers=$profile->get_headers();
-	if (strpos($headers['chost'], 'x86_64') === false)
-		$minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso';
-	else
-		$minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso';
-	makedir("$I/boot");
-	execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'");
-	file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n");
-	execute_command('Extract kernel modules from SquashFS', "unsquashfs -i -d '$W/modules' -e '$W/unsquashfs-files' '$W/image.squashfs'");
-	execute_command('Copy extracted modules to image', "cp -av '$W/modules/*' '$I/'");
-execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/image.squashfs' -noappend -info");
+	emerge('app-misc/livecd-toold', 'Install LiveCD utilities');
+	require(dirname(__FILE__).'/cd.inc');
+	execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/image.squashfs' -noappend -info");
 	execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' replace /image.squashfs '$W/image.squashfs' write '$W/image.iso'");
 	return "$W/image.iso";
 }
diff --git a/backend/functions/api.php b/backend/functions/api.php
index 1a0cc43..d8d648c 100644
--- a/backend/functions/api.php
+++ b/backend/functions/api.php
@@ -97,4 +97,8 @@ function makedir($dir) {
 	if (!is_dir($dir))
 		log_status('Create '.$dir, mkdir($dir, 0700, true));
 }
+function add_step($step) {
+	global $S;
+	$S['build_steps'][]=$step;
+}
 ?>
diff --git a/backend/functions/order_management.php b/backend/functions/order_management.php
index 306e42d..b7609f6 100644
--- a/backend/functions/order_management.php
+++ b/backend/functions/order_management.php
@@ -7,7 +7,11 @@ function task_get_order() {
 		$order++;
 	} else {
 		$buildid=$build->id;
-		$order=0;
+		$order=query('SELECT MAX(`order`) FROM `tasks` WHERE `build`="'.$buildid.'"')->fetch(PDO::FETCH_COLUMN);
+		if ($order === null)
+			$order=0;
+		else
+			$order++;
 	}
 	return $order;
 }
diff --git a/backend/functions/signals.php b/backend/functions/signals.php
index 2a895e5..70231c7 100644
--- a/backend/functions/signals.php
+++ b/backend/functions/signals.php
@@ -13,7 +13,8 @@ function handle_signal($sig=null) {
 			debug('$task not set');
 		}
 		$build->finish=time();
-		$build->status=$sig;
+		$build->status='got_signal';
+		$build->details=$sig;
 		$build->write();
 		debug("build $build->id given status $build->status");
 	}
diff --git a/backend/include/includes.php b/backend/include/includes.php
new file mode 100644
index 0000000..d20c3ac
--- /dev/null
+++ b/backend/include/includes.php
@@ -0,0 +1,4 @@
+<?php
+foreach (glob(BACKEND.'/bundlers/*.php') as $file)
+	require_once($file);
+?>
diff --git a/backend/modules/gentoo_portage/base-system.php b/backend/modules/gentoo_portage/base-system.php
index 6cf2194..c99447e 100644
--- a/backend/modules/gentoo_portage/base-system.php
+++ b/backend/modules/gentoo_portage/base-system.php
@@ -1,10 +1,10 @@
 <?php
 $file=CACHE.'/stage3/'.$profile->stage3;
 execute_command('Unpack base system', "tar -xvjpf '$file' -C '$I'");
-if ($opts['basesystem'] == 'manual' && $opts['prunepkgs']) {
+if ($opts['basesystem'] == 'user_prune' && $opts['prunepkgs']) {
 	emerge($opts['prunepkgs'], 'Prune base system packages', '-C');
-} elseif ($opts['basesystem'] == 'autoprune') {
-	throw_exception('Base system autoprune not implemented - need package list');
+} elseif ($opts['basesystem'] == 'auto_prune') {
+	throw_exception('Base system auto-prune not implemented - need package list');
 	$keep=explode(' ', $keep_pkgs);
 	$remove=array();
 	$r=query('SELECT * FROM `gentoo_basepkgs` WHERE `profile`='.$profile->id);
@@ -16,5 +16,7 @@ if ($opts['basesystem'] == 'manual' && $opts['prunepkgs']) {
 			unset($keep[$i]);
 	}
 	emerge($remove, 'Automatically prune base system packages', '-C');
+} elseif ($opts['basesystem'] == 'emerge') {
+	emerge('system', 'Emerge base system');
 }
 ?>
diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php
index 631cf0f..781aaef 100644
--- a/backend/modules/gentoo_portage/build.php
+++ b/backend/modules/gentoo_portage/build.php
@@ -1,37 +1,26 @@
 <?php
-require_once(dirname(__FILE__).'/packages.php'); // __DIR__ 5.3.0	
-// This is the main function that carries out a build from start to finish
-function gentoo_portage_build(&$build, &$opts, &$W) {
-	global $S;
-	$profile=new sql_gentoo_profile($opts['profile']);
-	$headers=$profile->get_headers();
-	$I="$W/image";
-	require(dirname(__FILE__).'/setup.php'); // __DIR__ in 5.3.0
-	if ($S['conf']['debug'])
-		execute_command_with_env('Log portage setup', 'emerge --info', $prtg_cfgrt);
-	require(dirname(__FILE__).'/base-system.php'); // __DIR__ 5.3.0
-	$extra=explode(' ', $opts['options']);
-	if (in_array('portage', $extra))
-		require(dirname(__FILE__).'/portage.php'); // __DIR__ 5.3.0
-	if (in_array('pruneinit', $extra))
-		require(dirname(__FILE__).'/init.d.php'); // __DIR__ 5.3.0
-	if (in_array('timezone', $extra))
-		require(dirname(__FILE__).'/timezone.php'); // __DIR__ 5.3.0
-	if (in_array('hostname', $extra))
-		require(dirname(__FILE__).'/hostname.php'); // __DIR__ 5.3.0
-	if (in_array('dev-manager', $extra))
-		require(dirname(__FILE__).'/dev-manager.php'); // __DIR__ 5.3.0
-	if ($opts['bundler'] == 'livecd')
-		emerge('app-misc/livecd-tools', 'Install LiveCD utilities');
-	if (strlen($opts['pkgsets'])) {
-		foreach (explode(' ', $opts['pkgsets']) as $pkgset) {
-			if (strlen($opts['pkgset-'.$pkgset])) {
-				emerge($opts['pkgset-'.$pkgset]);
-			}
-		}
-	}
-	if (strlen($opts['install_packages']))
-		emerge($opts['install_packages'], 'Install selected packages');
-	return $I;
-}
+require_once(dirname(__FILE__).'/packages.php'); // __DIR__ 5.3.0
+$profile=new sql_gentoo_profile($opts['profile']);
+$headers=$profile->get_headers();
+$I="$workdir/image";
+$C="$workdir/config_root";
+$S['prtg_cfgrt']=array('PORTAGE_CONFIGROOT' => $C);
+add_step('setup');
+add_step('base-system');
+$extra=explode(' ', $opts['options']);
+if (in_array('portage', $extra))
+	add_step('portage');
+if (in_array('prune_init', $extra))
+	add_step('init.d.php');
+if (in_array('timezone', $extra))
+	add_step('timezone');
+if (in_array('hostname', $extra))
+	add_step('hostname');
+if (in_array('dev-manager', $extra))
+	add_step('dev-manager');
+if (strlen($opts['pkgsets']))
+	add_step('pkgsets');
+if (strlen($opts['install_packages']))
+	add_step('misc-pkgs');
+return $I;
 ?>
diff --git a/backend/modules/gentoo_portage/misc-pkgs.php b/backend/modules/gentoo_portage/misc-pkgs.php
new file mode 100644
index 0000000..2c5185f
--- /dev/null
+++ b/backend/modules/gentoo_portage/misc-pkgs.php
@@ -0,0 +1,3 @@
+<?php
+emerge($opts['install_packages'], 'Install selected packages');
+?>
diff --git a/backend/modules/gentoo_portage/packages.php b/backend/modules/gentoo_portage/packages.php
index 167ca73..0274f35 100644
--- a/backend/modules/gentoo_portage/packages.php
+++ b/backend/modules/gentoo_portage/packages.php
@@ -1,6 +1,6 @@
 <?php
 function emerge($pkgs, $desc=null, $opts=null, $use=null) {
-	global $prtg_cfgrt, $S;
+	global $S;
 	$opts=$S['conf']['emerge_default_opts'].' '.($opts === null?'-K -n --root-deps=rdeps':$opts);
 	if (!$pkgs) return;
 	if (!is_array($pkgs))
@@ -9,6 +9,6 @@ function emerge($pkgs, $desc=null, $opts=null, $use=null) {
 		$desc='Install '.(count($pkgs) > 1?'packages':$pkgs[0]);
 	foreach ($pkgs as $i => &$pkg)
 		$pkg=escapeshellarg($pkg);
-	execute_command_with_env($desc, ($use?'env USE="'.(is_array($use)?implode(' ', $use):$use).'" ':'').'emerge '.($opts?$opts.' ':'').implode(' ', $pkgs), $prtg_cfgrt);
+	execute_command_with_env($desc, ($use?'env USE="'.(is_array($use)?implode(' ', $use):$use).'" ':'').'emerge '.($opts?$opts.' ':'').implode(' ', $pkgs), $S['prtg_cfgrt']);
 }
 ?>
diff --git a/backend/modules/gentoo_portage/pkgsets.php b/backend/modules/gentoo_portage/pkgsets.php
new file mode 100644
index 0000000..c0ed4e7
--- /dev/null
+++ b/backend/modules/gentoo_portage/pkgsets.php
@@ -0,0 +1,7 @@
+<?php
+foreach (explode(' ', $opts['pkgsets']) as $pkgset) {
+	if (strlen($opts['pkgset-'.$pkgset])) {
+		emerge($opts['pkgset-'.$pkgset]);
+	}
+}
+?>
diff --git a/backend/modules/gentoo_portage/portage.php b/backend/modules/gentoo_portage/portage.php
index 291a858..a1eb3ed 100644
--- a/backend/modules/gentoo_portage/portage.php
+++ b/backend/modules/gentoo_portage/portage.php
@@ -18,16 +18,21 @@ if ($file) {
 	end_internal_task(0);
 }
 start_internal_task('Set up portage in image');
+$use=explode(' ', $headers['use']);
+foreach(explode(' ', shell_exec(dirname(__FILE__).'/profile_use.py '.escapeshellarg("{$S['conf']['portdir']}/profiles/{$headers['profile']}"))) as $flag)
+	if (($i=array_search($flag, $use)) !== false)
+		unset($use[$i]);
 $makeconf=array(
 	'chost' => $headers['chost'],
 	'accept_keywords' => $headers['accept_keywords'],
 	'gentoo_mirrors' => $headers['gentoo_mirrors'],
-	'use' => $headers['use']
+	'use' => implode(' ', $use)
 );
 $contents='';
 foreach ($makeconf as $name => $val)
-	$contents.=strtoupper($name).'='.escapeshellarg($val)."\n";
+	$contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n";
 unset($makeconf);
+log_msg("/etc/make.conf:\n$contents");
 log_status('Writing /etc/make.conf', file_put_contents("$I/etc/make.conf", $contents));
 log_status('Remove previous make.profile', unlink("$I/etc/make.profile"), false);
 log_status("Symlink make.profile -> /usr/portage/profiles/{$headers['profile']}", symlink("/usr/portage/profiles/{$headers['profile']}", "$I/etc/make.profile"));
diff --git a/backend/modules/gentoo_portage/setup.php b/backend/modules/gentoo_portage/setup.php
index 5738bee..d96eb51 100644
--- a/backend/modules/gentoo_portage/setup.php
+++ b/backend/modules/gentoo_portage/setup.php
@@ -1,25 +1,22 @@
 <?php
 start_internal_task('Create portage target environment');
-$C=$W.'/config_root';
-makedirs($I, $C, "$W/log", "$W/tmp");
-log_status("Making symlink $C/etc -> .", symlink('.', "$C/etc"));
+makedirs($I, $C, "$workdir/log", "$workdir/tmp");
+log_status("Make symlink $C/etc -> .", symlink('.', "$C/etc"));
 $makeconf=array(
 	'pkgdir' => $S['conf']['pkgdir_root'].'/'.$profile->pkgdir,
 	'chost' => $headers['chost'],
 	'accept_keywords' => $headers['accept_keywords'],
 	'root' => $I,
-	'port_logdir' => "$W/log",
-	'emerge_log_dir' => "$W/log",
-	'portage_tmpdir' => "$W/tmp"
+	'port_logdir' => "$workdir/log",
+	'emerge_log_dir' => "$workdir/log",
+	'portage_tmpdir' => "$workdir/tmp"
 );
 $contents='';
 foreach ($makeconf as $name => $val)
-	$contents.=strtoupper($name).'='.escapeshellarg($val)."\n";
+	$contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n";
 unset($makeconf);
-log_status("Writing $C/etc/make.conf", file_put_contents("$C/etc/make.conf", $contents));
+log_status("Write $C/etc/make.conf", file_put_contents("$C/etc/make.conf", $contents));
 unset($contents);
-log_status('Making make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile'));
-global $prtg_cfgrt;
-$prtg_cfgrt=array('PORTAGE_CONFIGROOT' => $C);
+log_status('Make make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile'));
 end_internal_task(0);
 ?>