Made bundling finished images separate from modules in backend; added metadata for frontend modules; added default config options and simplified config file; added verbose output to update_gentoo_profiles.php; etc.

29 files changed, 233 insertions, 134 deletions

diff --git a/backend/backend.php b/backend/backend.php
index d078191..5c77fc3 100755
--- a/backend/backend.php
+++ b/backend/backend.php
@@ -66,7 +66,7 @@ while (true) {
 				unset($opt);
 			}
 			$opts=$build->get_opts();
-			$module=$opts['backend_module'];
+			$module=$opts['module'];
 			$build_proc=$module.'_build';
 			foreach (glob(BACKEND."/modules/$module/*.php") as $inc) {
 				require_once($inc);
@@ -74,7 +74,10 @@ while (true) {
 			// TODO check that build_proc exists
 			$workdir=WORK.'/build-'.$build->id;
 			fatal(log_status('Creating work directory '.$workdir, mkdir($workdir, 0700)));
-			$file=$build_proc($build, $opts, $workdir);
+			$image=$build_proc($build, $opts, $workdir);
+			require_once(BACKEND."/bundlers/{$opts['bundler']}.php");
+			$proc='bundle_'.$opts['bundler'];
+			$file=$proc($image, $workdir, $opts);
 			if (!$conf['debug']) {
 				execute_command('Delete work directory', 'rm -rf "'.$workdir.'"');
 			}
diff --git a/backend/bundlers/ext2.php b/backend/bundlers/ext2.php
new file mode 100644
index 0000000..239b5bb
--- /dev/null
+++ b/backend/bundlers/ext2.php
@@ -0,0 +1,12 @@
+<?php
+function bundle_ext2($I, $W) {
+	execute_command('Make blank file for ext2 image', "dd if=/dev/zero of='$W/image.ext2' bs=1024 count=1048576");
+	execute_command('Make ext2 filesystem', "mke2fs -t ext2 -F '$W/image.ext2'");
+	makedir('ext2');
+	execute_command('Mount ext2 image', "mount -o loop -t ext2 '$W/image.ext2' '$W/ext2'");
+	execute_command('Copy files to ext2', "cp -va '$I/*' '$W/ext2/'");
+	execute_command('Unmount ext2 image', "umount '$W/ext2'");
+	execute_command('Compress ext2 image', "gzip '$W/image.ext2'");
+	return "$W/image.ext2.gz";
+}
+?>
diff --git a/backend/bundlers/installcd.php b/backend/bundlers/installcd.php
new file mode 100644
index 0000000..90d549d
--- /dev/null
+++ b/backend/bundlers/installcd.php
@@ -0,0 +1,21 @@
+<?php
+function bundle_installcd($I, $W, &$opts) {
+	$profile=new sql_gentoo_profile($opts);
+	$headers=$profile->get_headers();
+	if (strpos($headers['chost'], 'x86_64') === false)
+		$minimaliso='/home/eitan/soc/install-x86-minimal-20090623.iso';
+	else
+		$minimaliso='/home/eitan/soc/install-amd64-minimal-20090625.iso';
+	execute_command('Mount minimal CD image', "mount -o loop -t iso9660 '$minimaliso' '$W/tmp'");
+	execute_command('Copy CD image to writable temp directory', "cp -va '$W/tmp' '$W/cd'");
+	execute_command('Unmount CD image', "umount '$W/tmp'");
+	execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$I/boot/'");
+	file_put_contents("$W/unsquashfs-files", "/lib64/modules\n\lib\modules\n");
+	execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$I' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'");
+	execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$I' .");
+	// TODO port the rest of /usr/lib/catalyst/targets/support/create-iso.sh to support other bootloaders
+	// ISOLINUX bootloader
+	// mkisofs -J -R -l ${mkisofs_zisofs_opts} -V "${clst_iso_volume_id}" -o ${1} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table ${clst_target_path}
+	execute_command('Create ISO image', "mkisofs -J -R -l -V 'Ingenue Build $build->id' -o '$W/image.iso' -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table '$W/cd'");
+	return "$W/image.iso";
+}
diff --git a/backend/bundlers/jffs2.php b/backend/bundlers/jffs2.php
new file mode 100644
index 0000000..c8e5de2
--- /dev/null
+++ b/backend/bundlers/jffs2.php
@@ -0,0 +1,6 @@
+<?php
+function bundle_jffs2($I, $W) {
+	execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$I' -o '$W/image.jffs2'");
+	return "$W/image.jffs2";
+}
+?>
diff --git a/backend/bundlers/livecd.php b/backend/bundlers/livecd.php
new file mode 100644
index 0000000..6bee4fc
--- /dev/null
+++ b/backend/bundlers/livecd.php
@@ -0,0 +1,23 @@
+<?php
+function bundle_livecd($I, $W, &$opts) {
+	$profile=new sql_gentoo_profile($opts);
+	$headers=$profile->get_headers();
+	if (strpos($headers['chost'], 'x86_64') === false)
+		$minimaliso='/home/eitan/soc/install-x86-minimal-20090623.iso';
+	else
+		$minimaliso='/home/eitan/soc/install-amd64-minimal-20090625.iso';
+	execute_command('Mount minimal CD image', "mount -o loop -t iso9660 '$minimaliso' '$W/tmp'");
+	execute_command('Copy CD image to writable temp directory', "cp -va '$W/tmp' '$W/cd'");
+	execute_command('Unmount CD image', "umount '$W/tmp'");
+	execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$I/boot/'");
+	file_put_contents("$W/unsquashfs-files", "/lib64/modules\n\lib\modules\n");
+	execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$I' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'");
+	rename("$W/cd/image.squashfs", "$W/image.squashfs.old") || debug('Failed to move old SquashFS');
+	execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/cd/image.squashfs' -noappend -info");
+	// TODO port the rest of /usr/lib/catalyst/targets/support/create-iso.sh to support other bootloaders
+	// ISOLINUX bootloader
+	// mkisofs -J -R -l ${mkisofs_zisofs_opts} -V "${clst_iso_volume_id}" -o ${1} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table ${clst_target_path}
+	execute_command('Create ISO image', "mkisofs -J -R -l -V 'Ingenue Build $build->id' -o '$W/image.iso' -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table '$W/cd'");
+	return "$W/image.iso";
+}
+?>
diff --git a/backend/bundlers/tbz2.php b/backend/bundlers/tbz2.php
new file mode 100644
index 0000000..ee69825
--- /dev/null
+++ b/backend/bundlers/tbz2.php
@@ -0,0 +1,6 @@
+<?php
+function bundle_tbz2($I, $W) {
+	execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' .");
+	return "$W/image.tar.bz2";
+}
+?>
diff --git a/backend/bundlers/tgz.php b/backend/bundlers/tgz.php
new file mode 100644
index 0000000..3b604d3
--- /dev/null
+++ b/backend/bundlers/tgz.php
@@ -0,0 +1,6 @@
+<?php
+function bundle_tgz($I, $W) {
+	execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$I' .");
+	return "$W/image.tar.gz";
+}
+?>
diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php
index aa8e8b5..7d8b1bc 100644
--- a/backend/modules/gentoo_portage/build.php
+++ b/backend/modules/gentoo_portage/build.php
@@ -48,48 +48,6 @@ function gentoo_portage_build(&$build, &$opts, &$W) {
 		foreach ($pkgs as $atom)
 			execute_command_with_env('Install extra package '.$atom, 'emerge '.$atom, $prtg_cfgrt);
 	}
-	$imgtype=isset($opts['image_type'])?$opts['image_type']:'tbz2';
-	if ($imgtype == 'tbz2') {
-		execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' .");
-		return "$W/image.tar.bz2";
-	} elseif ($imgtype == 'tgz') {
-		execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$I' .");
-		return "$W/image.tar.gz";
-	} elseif ($imgtype == 'livecd' || $imgtype == 'installcd') {
-		if (strpos($headers['chost'], 'x86_64') === false)
-			$minimaliso='/home/eitan/soc/install-x86-minimal-20090623.iso';
-		else
-			$minimaliso='/home/eitan/soc/install-amd64-minimal-20090625.iso';
-		execute_command('Mount minimal CD image', "mount -o loop -t iso9660 '$minimaliso' '$W/tmp'");
-		execute_command('Copy CD image to writable temp directory', "cp -va '$W/tmp' '$W/cd'");
-		execute_command('Unmount CD image', "umount '$W/tmp'");
-		execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$I/boot/'");
-		file_put_contents("$W/unsquashfs-files", "/lib64/modules\n\lib\modules\n");
-		execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$I' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'");
-		if ($imgtype == 'livecd') {
-			rename("$W/cd/image.squashfs", "$W/image.squashfs.old") || debug('Failed to move old SquashFS');
-			execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/cd/image.squashfs' -noappend -info");
-		} else // Install CD
-			execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$I' .");
-		// TODO port the rest of /usr/lib/catalyst/targets/support/create-iso.sh to support other bootloaders
-		// ISOLINUX bootloader
-		// mkisofs -J -R -l ${mkisofs_zisofs_opts} -V "${clst_iso_volume_id}" -o ${1} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table ${clst_target_path}
-		execute_command('Create ISO image', "mkisofs -J -R -l -V 'Ingenue Build $build->id' -o '$W/image.iso' -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table '$W/cd'");
-		return "$W/image.iso";
-	} elseif ($imgtype == 'jffs2') {
-		execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$I' -o '$W/image.jffs2'");
-		return "$W/image.jffs2";
-	} elseif ($imgtype == 'ext2') {
-		execute_command('Make blank file for ext2 image', "dd if=/dev/zero of='$W/image.ext2' bs=1024 count=1048576");
-		execute_command('Make ext2 filesystem', "mke2fs -t ext2 -F '$W/image.ext2'");
-		makedir('ext2');
-		execute_command('Mount ext2 image', "mount -o loop -t ext2 '$W/image.ext2' '$W/ext2'");
-		execute_command('Copy files to ext2', "cp -va '$I/*' '$W/ext2/'");
-		execute_command('Unmount ext2 image', "umount '$W/ext2'");
-		execute_command('Compress ext2 image', "gzip '$W/image.ext2'");
-		return "$W/image.ext2.gz";
-	} else {
-		throw_exception('invalid image type: '.$imgtype);
-	}
+	return $I;
 }
 ?>
diff --git a/frontend/classes/module.php b/frontend/classes/module.php
new file mode 100644
index 0000000..8971c42
--- /dev/null
+++ b/frontend/classes/module.php
@@ -0,0 +1,12 @@
+<?php
+class module {
+	var $id, $name, $steps, $dir;
+	function __construct($id) {
+		$this->id=$id;
+		require(FRONTEND."/modules/$id.info");
+		$this->name=$name;
+		$this->steps=$steps;
+		$this->dir=FRONTEND.'/modules/'.$dir;
+	}
+}
+?>
diff --git a/frontend/classes/wizard_api.php b/frontend/classes/wizard_api.php
index b23b8a5..a3b6fc8 100644
--- a/frontend/classes/wizard_api.php
+++ b/frontend/classes/wizard_api.php
@@ -2,18 +2,21 @@
 class wizard_step {
 	var $module, $step, $title, $next, $data=array();
 	function __construct($mod, $step) {
-		$this->module=$mod;
+		$this->module=new module($mod);
 		$this->step=$step;
-		$file=FRONTEND."/modules/$mod/step$step.php";
+		$file=$this->module->dir."/step$step.php";
 		if (!is_readable($file)) {
 			throw_exception("$mod step $step doesn't exist!");
 		}
 		require($file);
-		$this->title="Step $step".($title?" - $title":'');
-		$this->next=isset($next)?$next:$step+1;
+		$this->title="Step $step/{$this->module->steps}".($title?" - $title":'');
+		$this->next=isset($next)?$next:($this->step == $this->module->steps?null:$step+1);
 	}
 	public function output() {
+		global $conf;
 		echo "<h3>$this->title</h3>\n";
+		$scale=$conf['progressbar_width']/$this->module->steps;
+		echo '<img src="'.url('images/full.gif').'" style="border-left: 1px solid black; border-top: 1px solid black; border-bottom: 1px solid black; width: '.$this->step*$scale.'px; height: 15px" /><img src="'.url('images/empty.gif').'" style="border-right: 1px solid black; border-top: 1px solid black; border-bottom: 1px solid black; width: '.($this->module->steps-$this->step)*$scale.'px; height: 15px" /><br/>'."\n";
 		echo '<form action="'.url('config/'.wizard::$configuration->id).'" method="post">';
 		foreach ($this->data as $obj) {
 			if (!$obj->status) {
diff --git a/frontend/images/empty.gif b/frontend/images/empty.gif
new file mode 100755
index 0000000..cbcda5d
--- /dev/null
+++ b/frontend/images/empty.gif
diff --git a/frontend/images/full.gif b/frontend/images/full.gif
new file mode 100755
index 0000000..cfe77aa
--- /dev/null
+++ b/frontend/images/full.gif
diff --git a/frontend/modules/gentoo/step1.php b/frontend/modules/gentoo/step1.php
index d3ee3b9..2b7a9f2 100644
--- a/frontend/modules/gentoo/step1.php
+++ b/frontend/modules/gentoo/step1.php
@@ -8,5 +8,4 @@ while ($profile=$r->fetch(PDO::FETCH_ASSOC)) {
 	$opts[$profile->id]=$display;
 }
 $this->select('profile', 'profile', 'Profile', $opts);
-$next=2;
 ?>
diff --git a/frontend/modules/gentoo/step2.php b/frontend/modules/gentoo/step2.php
index df8ecce..da023c6 100644
--- a/frontend/modules/gentoo/step2.php
+++ b/frontend/modules/gentoo/step2.php
@@ -11,41 +11,4 @@ $meta=array(
 	array('tag' => 'div', 'label' => '=%p', 'checkbox' => '=%p', 'search' => true)
 );
 $this->layered_checkbox_array('install_packages', 'p', null, $pkgs, ' ', $meta);
-/*function gentoo_body_step2() {
-	global $S;
-	$configuration=&$S['wizard']['configuration'];
-	$opts=$configuration->get_configopts();
-	$profile=new sql_gentoo_profile($opts['profile']);
-	$bcs=$profile->get_packages();
-	echo 'Search packages: <input id="psearch" onkeyup="packagesearch(this)" /> <a href="javascript:var e=document.getElementById(\'psearch\'); e.value=\'\'; e.onkeyup()">Clear</a>';
-	echo '<div id="plist"><div id="zero" style="display: none">'.print_error('No results found.').'</div>';
-	$i=0;
-	foreach ($bcs as $bc => $lcs) {
-		echo '<div class="bct" id="bct-'.$bc.'">'.htmlentities($bc).'</div><div class="bc" id="bc-'.$bc.'">'."\n";
-		foreach ($lcs as $lc => $packages) {
-			echo '<div class="lct" id="lct-'.$bc.$lc.'"><a href="javascript:tog_show_pkgs(\''.$bc.$lc.'\')">&plusmn;</a> '.htmlentities("$bc$lc").'</div><div class="lc" id="lc-'.$bc.$lc.'" style="display: none">'."\n";
-			foreach ($packages as $name => $vers) {
-				foreach ($vers as $ver => $attrs) {
-					$safename=htmlentities("$bc$lc/$name-$ver");
-					echo '<div class="pkg'.($attrs['masked']?' masked':'').'"><input id="p'.$i.'" type="checkbox" name="expkgs['.$safename.']" /><label class="pd" for="p'.$i++.'">'.htmlentities("$name-$ver").' - '.htmlentities($attrs['desc']).($attrs['masked']?' [MASKED]':'').'</label></div>'."\n";
-				}
-			}
-			echo '</div>'."\n";
-		}
-		echo '</div>'."\n";
-	}
-	echo '</div>';
-}
-function gentoo_process_step2() {
-	global $S, $request;
-	if (isset($request['expkgs'])) {
-		$packages=array();
-		foreach ($request['expkgs'] as $name => $null) {
-			$packages[]='='.$name;
-		}
-		$packages=implode(' ', $packages);
-		$opt=new sql_configopt($S['wizard']['configuration']->id, 'install_packages', $packages);
-		$opt->write();
-	}
-}*/
 ?>
diff --git a/frontend/modules/gentoo/step3.php b/frontend/modules/gentoo/step3.php
index 9dd523a..0643856 100644
--- a/frontend/modules/gentoo/step3.php
+++ b/frontend/modules/gentoo/step3.php
@@ -1,7 +1,8 @@
 <?php
-// TODO This shouldn't be a step at all, it should be in wizard.php to choose between output modules
+// TODO This shouldn't be a step at all, it should be in wizard.php to choose between bundlers
+// TODO This shouldn't be part of configurations, except possibly a default value.  It should be for builds
 $title='Image Format';
-$this->select('image_type', 'image_type', 'Image type', array(
+$this->select('bundler', 'bundler', 'Image type', array(
 	'tgz' => 'Tar/Gzip',
 	'tbz2' => 'Tar/Bzip2',
 	'installcd' => 'Installer CD with Tar/Bzip2',
@@ -9,5 +10,4 @@ $this->select('image_type', 'image_type', 'Image type', array(
 	'ext2' => 'ext2',
 	'jffs2' => 'jffs2'
 ));
-$next=null;
 ?>
diff --git a/frontend/modules/gentoo_catalyst b/frontend/modules/gentoo_catalyst
deleted file mode 120000
index 0c0273e..0000000
--- a/frontend/modules/gentoo_catalyst
+++ /dev/null
@@ -1 +0,0 @@
-gentoo
\ No newline at end of file
diff --git a/frontend/modules/gentoo_catalyst.info.alpha b/frontend/modules/gentoo_catalyst.info.alpha
new file mode 100644
index 0000000..4803759
--- /dev/null
+++ b/frontend/modules/gentoo_catalyst.info.alpha
@@ -0,0 +1,4 @@
+<?php
+$name='Gentoo Catalyst';
+require(dirname(__FILE__).'/gentoo_common.php');
+?>
diff --git a/frontend/modules/gentoo_common.php b/frontend/modules/gentoo_common.php
new file mode 100644
index 0000000..cac9b4b
--- /dev/null
+++ b/frontend/modules/gentoo_common.php
@@ -0,0 +1,4 @@
+<?php
+$steps=3;
+$dir='gentoo';
+?>
diff --git a/frontend/modules/gentoo_portage b/frontend/modules/gentoo_portage
deleted file mode 120000
index 0c0273e..0000000
--- a/frontend/modules/gentoo_portage
+++ /dev/null
@@ -1 +0,0 @@
-gentoo
\ No newline at end of file
diff --git a/frontend/modules/gentoo_portage.info b/frontend/modules/gentoo_portage.info
new file mode 100644
index 0000000..cb6e198
--- /dev/null
+++ b/frontend/modules/gentoo_portage.info
@@ -0,0 +1,4 @@
+<?php
+$name='Gentoo Portage';
+require(dirname(__FILE__).'/gentoo_common.php');
+?>
diff --git a/frontend/pages/configurations/manager.php b/frontend/pages/configurations/manager.php
index 21ee9dc..aef03d1 100644
--- a/frontend/pages/configurations/manager.php
+++ b/frontend/pages/configurations/manager.php
@@ -7,7 +7,7 @@ function init_configurations_manager() {
 	return array('title' => 'Manage Configurations');
 }
 function body_configurations_manager() {
-	global $S, $request;
+	global $S, $request, $conf;
 	echo '<h3>Configurations Manager</h3>';
 	$r=$S['pdo']->query('SELECT * FROM `configurations` WHERE `owner`='.$S['user']->id);
 	if ($r->rowCount() == 0) {
@@ -23,10 +23,29 @@ function body_configurations_manager() {
 			echo print_success('Submitted for build - <a href="'.url('logs/'.$build->id).'">Logs</a>');
 		}
 	}
-	echo '<form action="'.url('configurations').'" method="post"><table><tr><th>ID</th><th>Name</th><th>Status</th><th>Options</th><th>Builds</th></tr>'."\n";
+	echo '<form action="'.url('configurations').'" method="post"><table><tr><th>ID</th><th>Name</th>'.(count(explode(' ', $conf['modules'])) > 1?'<th>Module</th>':'').'<th>Status</th><th>Options</th><th>Builds</th></tr>'."\n";
+	$ready=0;
 	while($c=$r->fetch(PDO::FETCH_ASSOC)) {
 		$c=new sql_configuration($c);
-		echo "<tr><td><input id=\"radio-$c->id\" type=\"radio\" name=\"configuration\" value=\"$c->id\" /> <label for=\"radio-$c->id\">$c->id</label></td><td>".(isset($c->name) && strlen($c->name)?htmlentities($c->name):'<i>Unnamed</i>')."</td><td>$c->status</td><td>".$c->summary().'</td><td>';
+		echo "<tr><td>";
+		if ($c->status == 0) {
+			$ready++;
+			echo "<input id=\"radio-$c->id\" type=\"radio\" name=\"configuration\" value=\"$c->id\" /> <label for=\"radio-$c->id\">".$c->id.'</label>';
+		} else {
+			echo $c->id;
+		}
+		echo '</td><td>'.(isset($c->name) && strlen($c->name)?htmlentities($c->name):'<i>Unnamed</i>').'</td><td>';
+		if (count(explode(' ', $conf['modules'])) > 1) {
+			echo "$c->module</td><td>";
+		}
+		if ($c->status > 0) {
+			echo '<a href="'.url("config/$c->id")."\">Step $c->status</a>'";
+		} elseif ($c->status == 0) {
+			echo '<b>Ready</b>';
+		} else {
+			echo $c->status;
+		}
+		echo '</td><td>'.$c->summary().'</td><td>';
 		$builds=$c->get_builds();
 		if ($builds) {
 			foreach ($builds as $build) {
@@ -37,6 +56,6 @@ function body_configurations_manager() {
 		}
 		echo "</td></tr>\n";
 	}
-	echo '</table>Name (optional): <input name="name" /> <input type="submit" name="build" value="Build" /></form>';
+	echo '</table>'.($ready?'Name (optional): <input name="name" /> <input type="submit" name="build" value="Build" />':'').'</form>';
 }
 ?>
diff --git a/frontend/pages/configurations/wizard.php b/frontend/pages/configurations/wizard.php
index de35c65..9a2b5d4 100644
--- a/frontend/pages/configurations/wizard.php
+++ b/frontend/pages/configurations/wizard.php
@@ -38,8 +38,7 @@ function init_configurations_wizard() {
 		wizard::set_configuration($S['wizard']['configuration']);		
 		$configuration=&$S['wizard']['configuration'];
 		$configuration->name=$request['name'];
-		$mods=explode(' ', $conf['modules']);
-		$mod=isset($request['mod']) && isset($mods[$request['mod']])?$mods[$request['mod']]:$mods[0];
+		$mod=isset($request['mod']) && isset($conf['modules'][$request['mod']])?$conf['modules'][$request['mod']]:$conf['modules'][0];
 		$configuration->module=$mod;
 		$configuration->init();
 		debug('wizard', "Module: $mod");
@@ -62,12 +61,13 @@ function body_configurations_wizard() {
 		}
 	} else {
 		echo '<form action="'.url('create').'" method="post"><h3>Request an image built</h3>Name of your configuration (optional): <input name="name" /><br/>';
-		$mods=explode(' ', $conf['modules']);
-		if (count($mods) > 1) {
+		if (count($conf['modules']) > 1) {
 			echo 'Module: <select name="mod">';
 			$i=0;
-			foreach ($mods as $mod)
-				echo '<option value="'.$i++."\">$mod</option>";
+			foreach ($conf['modules'] as $mod) {
+				$mod=new module($mod);
+				echo '<option value="'.$i++.'">'.htmlentities($mod->name).'</option>';
+			}
 			echo '</select><br/>';
 		}
 		echo '<input type="submit" name="init" value="Start" /></form>';
diff --git a/shared/classes/gentoo_profile.php b/shared/classes/gentoo_profile.php
index b705a97..b2b970f 100644
--- a/shared/classes/gentoo_profile.php
+++ b/shared/classes/gentoo_profile.php
@@ -48,9 +48,12 @@ class sql_gentoo_profile extends sql_row_obj {
 		return $this->headers_cache;
 	}
 	// Reads the data from the Packages file in $this->pkgdir
-	public function read_Packages($update_pkgs=false) {
+	public function read_Packages($update_pkgs=false, $verbose=false) {
 		global $conf;
-		$file=fopen($conf['pkgdir_root'].'/'.$this->pkgdir.'/Packages', 'r');
+		if (!is_readable($file="{$conf['pkgdir_root']}/$this->pkgdir/Packages")) {
+			throw_exception("Packages file doesn't exist for pkgdir $this->pkgdir");
+		}
+		$file=fopen($file, 'r');
 		$this->headers='';
 		while (!feof($file)) {
 			$line=rtrim(fgets($file));
@@ -83,6 +86,7 @@ class sql_gentoo_profile extends sql_row_obj {
 				}
 				if (isset($p[$bcat][$lcat][$name][$ver])) {
 					debug("Duplicate package $bcat$lcat/$name-$ver");
+					continue;
 				}
 				$p[$bcat][$lcat][$name][$ver]='';
 				$cur=&$p[$bcat][$lcat][$name][$ver];
@@ -91,20 +95,24 @@ class sql_gentoo_profile extends sql_row_obj {
 			}
 		}
 		unset($cur);
-		$u=0;
+		$u=$d=$t=0;
 		if ($update_pkgs) {
 			global $S;
 			$r=$S['pdo']->query('SELECT * FROM `gentoo_packages` WHERE `profile`='.$this->id);
 			while ($pkg=$r->fetch(PDO::FETCH_ASSOC)) {
 				$pkg=new sql_gentoo_package($pkg);
 				if (isset($p[$pkg->bcat][$pkg->lcat][$pkg->name][$pkg->version])) {
+					$t++;
 					if ($pkg->data != $p[$pkg->bcat][$pkg->lcat][$pkg->name][$pkg->version]) {
 						$u++;
 						$pkg->data=$p[$pkg->bcat][$pkg->lcat][$pkg->name][$pkg->version];
+						if ($verbose) echo "U $pkg->bcat$pkg->lcat/$pkg->name/$pkg->version\n";
 						$pkg->write();
 					}
 					unset($p[$pkg->bcat][$pkg->lcat][$pkg->name][$pkg->version]);
 				} else {
+					$d++;
+					if ($verbose) echo "D $pkg->bcat$pkg->lcat/$pkg->name-$pkg->version\n";
 					$pkg->delete();
 				}
 			}
@@ -114,14 +122,16 @@ class sql_gentoo_profile extends sql_row_obj {
 			foreach ($lcats as $lcat => $pkgs) {
 				foreach ($pkgs as $pkg => $vers) {
 					foreach ($vers as $ver => $data) {
+						$t++;
 						$n++;
 						$gp=new sql_gentoo_package(null, $this->id, $bcat, $lcat, $pkg, $ver, $data);
+						if ($verbose) echo "A $bcat$lcat/$pkg-$ver\n";
 						$gp->write();
 					}
 				}
 			}
 		}
-		return array($n, $u);
+		return array($n, $u, $d, $t);
 	}
 	public function &get_packages() {
 		global $S;
diff --git a/shared/config.php b/shared/config.php
index b42cb8a..1ac1ae7 100644
--- a/shared/config.php
+++ b/shared/config.php
@@ -1,23 +1,26 @@
 <?php
-$conf['title']='Ingenue'; // Used in <title>
-$conf['sqlhost']='localhost'; // MySQL server
-$conf['sqluser']='soc'; // MySQL username
-$conf['sqlpass']='socpassword'; // MySQL password
-$conf['sqldb']='soc'; // MySQL database
-$conf['debug']=true; // Whether to print debugging information
-$conf['cache']=true; // Whether to enable built-in caching
-$conf['modules']='gentoo_portage x gentoo_catalyst'; // Space-separated list of modules to offer the user
-$conf['cookiename']='ingenueid'; // Name of the cookie to send for keeping sessions
-$conf['sessionlength']=1814400; // Time in seconds before sessions are purged
-$conf['timezone']=10800; // Time difference in seconds between UTC and the default timezone
-$conf['mod_rewrite']=true; // Use mod_rewrite for pretty URLs
-$conf['emailfrom']='noreply@gentoo.org'; // Used as the From: field in emails
-$conf['check_email_dns']=true; // Use DNS to check the domain of submitted emails for validity
-$conf['pkgdir_root']='/home/eitan/soc/tinderbox'; // The directory to recursively search for pkgdirs in (Backend only)
-$conf['emerge_default_opts']='-t -v -K --color=y --root-deps=rdeps'; // DON'T CHANGE UNLESS YOU KNOW WHAT YOU'RE DOING
-$conf['portdir']='/usr/portage'; // The directory conatining the portage tree to use (/usr/portage unless you have a reason to think otherwise)
-$conf['logview_max']=1000; // The maximum number of log entries shown on one page (1000 is a good start)
-$conf['split_setup']=true; // Whether the frontend and backend are running on different hosts
-$conf['frontend_location']='http://soc'; // The base address of the frontend installation (for use by the backend)
-$conf['backend_id']='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend TODO use gethostname() by default in 5.3.0
+// Default values are commented
+// $title='Ingenue'; // Used in <title> and in emails
+// $sqlhost='localhost'; // MySQL server
+$sqluser='soc'; // MySQL username
+$sqlpass='socpassword'; // MySQL password
+$sqldb='soc'; // MySQL database
+$debug=true; // Whether to print debugging information
+// $modules='All non-hidden dirs in frontend/modules'; // Space-separated list of modules to offer the user
+// $bundlers='All non-hidden <bundler>.php files in backend/bundlers'; // Space-separated list of bundlers to offer the user
+// $cookiename='ingenueid'; // Name of the cookie to send for keeping sessions
+// $sessionlength=1814400; // Time in seconds before sessions are purged
+// $mod_rewrite=true; // Use mod_rewrite for pretty URLs
+$emailfrom='noreply@gentoo.org'; // Used as the From: field in emails
+$check_email_dns=true; // Use DNS to check the domain of submitted emails for validity
+// $split_setup=false; // Whether the frontend and backend are running on different hosts
+// Frontend options:
+// $logview_max=1000; // The maximum number of log entries shown on one page (1000 is a good start)
+// $progressbar_width=350; // The width, in pixels, of the config wizard progress bar
+// Backend options:
+$pkgdir_root='/home/eitan/soc/tinderbox'; // The directory to recursively search for pkgdirs (Backend only)
+// $emerge_default_opts='-t -v -K --color=y --root-deps=rdeps'; // DON'T CHANGE UNLESS YOU KNOW WHAT YOU'RE DOING
+// $portdir='/usr/portage'; // The directory conatining the portage tree to use (/usr/portage unless you have a reason to think otherwise)
+$frontend_location='http://soc'; // The base address of the frontend installation
+$backend_id='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend TODO use gethostname() by default in 5.3.0
 ?>
diff --git a/shared/functions/load_config.php b/shared/functions/load_config.php
new file mode 100644
index 0000000..27e01bc
--- /dev/null
+++ b/shared/functions/load_config.php
@@ -0,0 +1,12 @@
+<?php
+function load_config() {
+	require(SHARED.'/include/defaults.php');
+	require(SHARED.'/config.php');	       	
+	$modules=explode(' ', $modules);
+	foreach (explode(' ', 'title sqlhost sqluser sqlpass sqldb debug modules bundlers cookiename sessionlength mod_rewrite emailfrom check_email_dns split_setup logview_max progressbar_width pkgdir_root emerge_default_opts portdir frontend_location backend_id') as $var) {
+		if (isset($$var)) {
+			$GLOBALS['conf'][$var]=$$var;
+		}
+	}
+}
+?>
diff --git a/shared/include/defaults.php b/shared/include/defaults.php
new file mode 100644
index 0000000..4065cc0
--- /dev/null
+++ b/shared/include/defaults.php
@@ -0,0 +1,32 @@
+<?php
+$title='Ingenue';
+$sqlhost='localhost';
+$sqluser=posix_getpwuid(posix_geteuid());
+$sqluser=$sqluser['name'];
+$sqlpass=null;
+$sqldb='ingenue';
+$debug=false;
+$modules=array();
+foreach (glob(FRONTEND.'/modules/*.info') as $module) {
+	$module=basename($module);
+	$modules[]=substr($module, 0, strlen($module)-5);
+}
+$modules=implode(' ', $modules);
+$bundlers=array();
+foreach (glob(BACKEND.'/bundlers/*.php') as $bundler) {
+	$bundler=basename($bundler);
+	$bundlers[]=substr($bundler, 0, strlen($bundler)-4);
+}
+$bundlers=implode(' ', $bundlers);
+$cookiename='ingenueid';
+$sessionlength=1814400;
+$mod_rewrite=true;
+$emailfrom='noreply@noreply.net';
+$check_email_dns=false;
+$split_setup=false;
+$logview_max=1000;
+$progressbar_width=350;
+$emerge_default_opts='-t -v -K --color=y --root-deps=rdeps';
+$portdir='/usr/portage';
+//$backend_id=gethostname(); // TODO Uncomment in 5.3.0
+?>
diff --git a/shared/include/includes.php b/shared/include/includes.php
index ae12d81..bfd358d 100644
--- a/shared/include/includes.php
+++ b/shared/include/includes.php
@@ -9,5 +9,5 @@ foreach (array('functions', 'classes') as $type) {
 	}
 }
 unset($dir, $file, $type);
-require_once(SHARED.'/config.php');
+load_config();
 ?>
diff --git a/todo b/todo
index c4401f4..94bf3ae 100644
--- a/todo
+++ b/todo
@@ -1,6 +1,6 @@
 Have backend handle builds that it finds to already be running (break in to steps and store current status)
 Make package adding friendly for browsers without JS/CSS (use ul/li, not div)
-Figure out how to adapt to newer versions of packages
+Add configuration verification - each step must be validated before a build is submitted (check selected packages are available, etc.)
 Write a live git ebuild
 Write an AJAX-based self-updating status viewer
 *** Add logging besides just commands ***
@@ -12,12 +12,13 @@ Add metadata back to logviewer
 Add configurable groups of suggested packages to frontend, backend
 Add a statistics page
 Add a profiles management page/backend utility
-Add cleanup functions to the frontend and backend
+Add cleanup functions to the frontend and backend (tasks dir in backend containing scripts that can be launched through frontend)
 Separate variables we got from the URL from the rest, stop using $request, instead keep super globals and strip slashes on them
-Support ~arch installation
+Add masked indicator back to step2, support ~arch installation
 Completely plan out how frontend modules should function - each step needs to report if it finished successfully (required values)
 Allow backend to define bail-out functions to call when it dies (things like unmounting the ISO it was copying)
 Add STDERR (maybe STDOUT) only option to log viewer
 Simplify status to numeric on builds - varchar isn't necessary
 Add config option to enable/disable user self-registration/invitations/admin-only invites
 Move gentoo_profiles setup out of the general setup.php, allow per-module setup
+Move bundler selection out of gentoo module and generalize it (switch to builds instead of configurations)
diff --git a/update_gentoo_profiles.php b/update_gentoo_profiles.php
index 545b08f..c2e9f20 100755
--- a/update_gentoo_profiles.php
+++ b/update_gentoo_profiles.php
@@ -7,8 +7,8 @@ $pdo=&$S['pdo'];
 $r=$pdo->query('SELECT * FROM `gentoo_profiles`'.(isset($argv[1])?' WHERE `name` LIKE "%'.$argv[1].'%" OR `pkgdir` LIKE "%'.$argv[1].'%"':''));
 while ($p=$r->fetch(PDO::FETCH_ASSOC)) {
 	$p=new sql_gentoo_profile($p);
-	echo 'Updating from '.$conf['pkgdir_root'].'/'.$p->pkgdir.'/Packages...';
-	list($new, $updated)=$p->read_Packages(true);
-	echo "done\nPackages: $new new, $updated updated.\n";
+	echo "Updating from {$conf['pkgdir_root']}/$p->pkgdir/Packages...\n";
+	list($new, $updated, $deleted, $total)=$p->read_Packages(true, true);
+	echo "Packages: $new new, $updated updated, $deleted deleted, $total total.\n";
 }
 ?>