Backend can upload finished images; caches state after emerge system completes

12 files changed, 111 insertions, 26 deletions

diff --git a/backend/backend.php b/backend/backend.php
index cde76f0..2ee7357 100755
--- a/backend/backend.php
+++ b/backend/backend.php
@@ -56,7 +56,7 @@ while (true) {
 		$build->status='build/running';
 		$build->write();
 		log_msg('Starting build id='.$build->id);
-		$success=null;
+		$file=null;
 		try {
 			$opts=$build->get_buildopts();
 			$module=$opts['backend_module'];
@@ -67,7 +67,7 @@ while (true) {
 			// TODO check that build_proc exists
 			$workdir=WORK.'/build-'.$build->id;
 			fatal(log_status('Creating work directory '.$workdir, mkdir($workdir, 0700)));
-			$success=$build_proc($build, $opts, $workdir);
+			$file=$build_proc($build, $opts, $workdir);
 			if (!$conf['debug']) {
 				execute_command('Delete work directory', 'rm -rf "'.$workdir.'"');
 			}
@@ -79,16 +79,46 @@ while (true) {
 		}
 		$build->finish=time();
 		log_msg('Finished with build id='.$build->id);
-		if (isset($success)) {
+		if (isset($file)) {
 			log_msg("Completed build successfully");
-			$build->status='finished/success';
+			if ($conf['split_setup']) {
+				$build->status='finished/uploading';
+				$build->write();
+				$key=randstring(30);
+				$opt=new sql_buildopt($build->id, 'uploadkey', $key);
+				$opt->write();
+				$c=curl_init($conf['frontend_location'].'/backend/upload_image');
+				curl_setopt($c, CURLOPT_POST, 1);
+				curl_setopt($c, CURLOPT_POSTFIELDS, array(
+					'build' => $build->id,
+					'key' => $key,
+					'file' => "@$file"
+				));
+				curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
+				$result=curl_exec($c);
+				if ($conf['debug'] && is_string($result)) {
+					debug($result);
+				}
+				if ($result === false || strpos($result, 'Upload successful') === false) {
+					$build->status='finished/failed: upload failed'.($result === false?' - '.curl_error($c):'');
+				} else {
+					debug("Transferred $file... unlinking it");
+					unlink($file);
+					$build->status='finished/success';
+				}
+			} else {
+				$build->status='finished/success';
+				$base=basename($file);
+				$ext=substr($base, substr($base, strpos($base, '.')));
+				rename($file, COMPLETED.'/build-'.$build->id.$ext);
+			}
 			xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $conf['title'].' build finished', 'Your build has completed successfully.  You can find more information and download the completed image at <a href="'.url('logs/'.$build->id).'">'.url('logs/'.$build->id).'</a>');
 		}
 		$build->write();
 		unset($build);
 	}
-//	log_msg('Sleeping...', false);
+	//	log_msg('Sleeping...', false);
 	sleep(5);
-//	log_msg("done");
+	//	log_msg("done");
 }
 ?>
diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php
index f23408f..c2d0be8 100644
--- a/backend/modules/gentoo_portage/build.php
+++ b/backend/modules/gentoo_portage/build.php
@@ -29,7 +29,13 @@ function gentoo_portage_build(&$build, &$opts, &$W) {
 	execute_command_with_env('Log portage setup', 'emerge --info', $prtg_cfgrt);
 //	execute_command_with_env('Pre-installing baselayout', 'emerge baselayout', array_merge($prtg_cfgrt, array('USE' => 'build')));
 	//	TODO create make.conf, make.profile in target /etc
-	execute_command_with_env('Install base system', 'emerge system', $prtg_cfgrt);
+	if (is_file(CACHE.'/system-'.$profile->id.'.tar.gz') && filemtime(CACHE.'/system-'.$profile->id.'.tar.gz') > filemtime($conf['pkgdir_root'].'/'.$profile->pkgdir.'/Packages')) {
+		execute_command('Unpack cached base system', "tar -zxvf '".CACHE."/system-$profile->id.tar.gz' -C '$I'");
+	} else {
+		execute_command_with_env('Install base system', 'emerge system', $prtg_cfgrt);
+		execute_command('Cache base system for reuse', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$I' .");
+		rename("$W/image.tar.gz", CACHE.'/system-'.$profile->id.'.tar.gz');
+	}
 	if (isset($opts['image_type']) && $opts['image_type'] == 'livecd')
 		execute_command_with_env('Install LiveCD utilities', 'emerge -1 livecd-tools', $prtg_cfgrt);
 	if (isset($opts['install_packages'])) {
@@ -44,11 +50,13 @@ function gentoo_portage_build(&$build, &$opts, &$W) {
 	}
 	$imgtype=isset($opts['image_type'])?$opts['image_type']:'tbz2';
 	if ($imgtype == 'tbz2') {
-		execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$W/image' .");
-		rename($W.'/image.tar.bz2', COMPLETED.'/build-'.$build->id.'.tar.bz2') || throw_exception('rename failed');
+		execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' .");
+		return "$W/image.tar.bz2";
+//		rename($W.'/image.tar.bz2', COMPLETED.'/build-'.$build->id.'.tar.bz2') || throw_exception('rename failed');
 	} elseif ($imgtype == 'tgz') {
-		execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$W/image' .");
-		rename($W.'/image.tar.gz', COMPLETED.'/build-'.$build->id.'.tar.gz') || throw_exception('rename failed');
+		execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$I' .");
+		return "$W/image.tar.gz";
+//		rename($W.'/image.tar.gz', COMPLETED.'/build-'.$build->id.'.tar.gz') || throw_exception('rename failed');
 	} elseif ($imgtype == 'livecd' || $imgtype == 'installcd') {
 		if (strpos($headers['chost'], 'x86_64') === false)
 			$minimaliso='/home/eitan/soc/install-x86-minimal-20090623.iso';
@@ -57,34 +65,37 @@ function gentoo_portage_build(&$build, &$opts, &$W) {
 		execute_command_with_env('Mount minimal CD image', "mount -o loop -t iso9660 '$minimaliso' '$W/tmp'", $prtg_cfgrt); // TODO check if env is necessary (and for umount)
 		execute_command('Copy CD image to writable temp directory', "cp -va '$W/tmp' '$W/cd'");
 		execute_command_with_env('Unmount CD image', "umount '$W/tmp'", $prtg_cfgrt);
-		execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$W/image/boot/'");
+		execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$I/boot/'");
 		file_put_contents("$W/unsquashfs-files", "/lib64/modules\n\lib\modules\n");
-		execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$W/image' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'");
+		execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$I' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'");
 		if ($imgtype == 'livecd') {
 			rename("$W/cd/image.squashfs", "$W/image.squashfs.old") || debug('Failed to move old SquashFS');
-			execute_command('Compress finished image to squashfs', "mksquashfs '$W/image' '$W/cd/image.squashfs' -noappend -info");
+			execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/cd/image.squashfs' -noappend -info");
 		} else // Install CD
-			execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$W/image' .");
+			execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$I' .");
 		// TODO port the rest of /usr/lib/catalyst/targets/support/create-iso.sh to support other bootloaders
 		// ISOLINUX bootloader
 		// mkisofs -J -R -l ${mkisofs_zisofs_opts} -V "${clst_iso_volume_id}" -o ${1} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table ${clst_target_path}
 		execute_command('Create ISO image', "mkisofs -J -R -l -V 'Ingenue Build $build->id' -o '$W/image.iso' -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table '$W/cd'");
-		rename("$W/image.iso", COMPLETED.'/build-'.$build->id.'.iso') || throw_exception('rename failed');
+		return "$W/image.iso";
+//		rename("$W/image.iso", COMPLETED.'/build-'.$build->id.'.iso') || throw_exception('rename failed');
 	} elseif ($imgtype == 'jffs2') {
-		execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$W/image' -o '$W/image.jffs2'");
-		rename("$W/image.jffs2", COMPLETED.'/build-'.$build->id.'.jffs2') || throw_exception ('rename failed');
+		execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$I' -o '$W/image.jffs2'");
+		return "$W/image.jffs2";
+//		rename("$W/image.jffs2", COMPLETED.'/build-'.$build->id.'.jffs2') || throw_exception ('rename failed');
 	} elseif ($imgtype == 'ext2') {
 		execute_command('Make blank file for ext2 image', "dd if=/dev/zero of='$W/image.ext2' bs=1024 count=1048576");
 		execute_command('Make ext2 filesystem', "mke2fs -t ext2 -F '$W/image.ext2'");
 		makedir('ext2');
 		execute_command('Mount ext2 image', "mount -o loop -t ext2 '$W/image.ext2' '$W/ext2'");
-		execute_command('Copy files to ext2', "cp -va '$W/image/*' '$W/ext2/'");
+		execute_command('Copy files to ext2', "cp -va '$I/*' '$W/ext2/'");
 		execute_command('Unmount ext2 image', "umount '$W/ext2'");
 		execute_command('Compress ext2 image', "gzip '$W/image.ext2'");
-		rename("$W/image.ext2.gz", COMPLETED.'/build-'.$build->id.'.ext2.gz') || throw_exception('rename failed');
+		return "$W/image.ext2.gz";
+//		rename("$W/image.ext2.gz", COMPLETED.'/build-'.$build->id.'.ext2.gz') || throw_exception('rename failed');
 	} else {
 		throw_exception('invalid image type: '.$imgtype);
 	}
-	return true;
+//	return true;
 }
 ?>
diff --git a/cache/.gitignore b/cache/.gitignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cache/.gitignore
diff --git a/completed/.gitignore b/completed/.gitignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/completed/.gitignore
diff --git a/frontend/.htaccess b/frontend/.htaccess
index 0fa2f57..c1804b1 100755
--- a/frontend/.htaccess
+++ b/frontend/.htaccess
@@ -1,4 +1,5 @@
+php_value upload_max_filesize 1073741824

+php_value post_max_size 1073742848

 RewriteEngine On

-#RewriteRule	^(recordings/[0-9]+\.mp3)$	$1	[L]

-RewriteRule	^(tmp(/.*)?)$	$1	[L]

+#RewriteRule	^(tmp(/.*)?)$	$1	[L]

 RewriteRule	^(?!main.php)(.*)$	index.php?req=$1&%{QUERY_STRING}

diff --git a/frontend/pages/404.php b/frontend/pages/404.php
index c81af31..e19c818 100644
--- a/frontend/pages/404.php
+++ b/frontend/pages/404.php
@@ -2,7 +2,7 @@
 function init_404() {
 	global $S;
 	$S['title']='404: Not Found';
-	header('HTTP/1.0 404 Not Found');
+	header('HTTP/1.0 404 Not Found', true, 404);
 }
 function body_404() {
 	global $S;
diff --git a/frontend/pages/upload.php b/frontend/pages/upload.php
new file mode 100644
index 0000000..0c5ceb3
--- /dev/null
+++ b/frontend/pages/upload.php
@@ -0,0 +1,40 @@
+<?php
+function init_upload() {
+	global $S, $request;
+	if (!(isset($request['build'], $request['key'], $_FILES['file']) && preg_match('/^[a-zA-Z0-9]{6}$/', $request['build']) && preg_match('/^[a-zA-Z0-9]{30}$/', $request['key']))) {
+		debug('upload', 'missing or malformed input');
+		return '404';
+	}
+	$r=$S['pdo']->query('SELECT * FROM `builds` WHERE `id`="'.$request['build'].'"');
+	if ($r->rowCount() == 0) {
+		debug('upload', 'build not found');
+		return '404';
+	}
+	$build=new sql_build($r->fetch(PDO::FETCH_ASSOC));
+	$opts=$build->get_buildopts();
+	if (!(isset($opts['uploadkey']) && $opts['uploadkey'] == $request['key'])) {
+		debug('upload', 'invalid upload key');
+		return '404';
+	}
+	debug('upload', 'error code: '.$_FILES['file']['error']);
+	if ($_FILES['file']['error'] != UPLOAD_ERR_OK) {
+		return '404';
+	}
+	debug('upload', 'Got uploaded file '.$_FILES['file']['name'].' at '.$_FILES['file']['tmp_name']);
+	$name=basename($_FILES['file']['name']);
+	$ext=substr($name, strpos($name, '.'));
+	debug('upload', $_FILES['file']['tmp_name'].' -> '.COMPLETED."/build-$build->id$ext");
+	if (!is_writable(COMPLETED)) {
+		debug('upload', 'Web server needs write permissions for '.COMPLETED);
+		return '404';
+	}
+	if (!move_uploaded_file($_FILES['file']['tmp_name'], COMPLETED."/build-$build->id$ext")) {
+		debug('Move file failed');
+		return '404';
+	}
+	return array('title' => 'Upload Successful');
+}
+function body_upload() {
+	echo print_success('Upload successful');
+}
+?>
diff --git a/frontend/routing.csv b/frontend/routing.csv
index bbec7d7..4b77f29 100644
--- a/frontend/routing.csv
+++ b/frontend/routing.csv
@@ -39,5 +39,7 @@
 #^(xinha(?=/)[0-9a-zA-Z-_./]*).(?<=/)([0-9a-zA-Z-_.]+\.([a-zA-Z0-9]+))$	passthrough	dir	file	ext
 # CSS
 ^style.css$	stylesheet
+# Backend access only
+^backend/upload_image$	upload
 # This is the catch-all - never remove it
 ^	404
diff --git a/shared/classes/build.php b/shared/classes/build.php
index 3d37ec3..9b2decc 100644
--- a/shared/classes/build.php
+++ b/shared/classes/build.php
@@ -100,6 +100,8 @@ class sql_build extends sql_row_obj {
 				$html.='<span class="status successful">[successful]</span><br/><span class="links"><a href="'.url('download/'.$this->id).'">Download image</a> &bull; <a href="'.url('logs/'.$this->id).'">Build log</a></span>';
 			} elseif ($status[0] == 'failed') {
 				$html.='<span class="status failed">[failed: '.htmlentities($status[1]).']</span><br/><span class="links"><a href="'.url('logs/'.$this->id.'/failure').'">View output of failed command</a> &bull; <a href="'.url('logs/'.$this->id).'">Build log</a></span>';
+			} elseif ($status[0] == 'uploading') {
+				$html.='<span class="status successful">[uploading]</span><br/><span class="links"><a href="'.url('logs/'.$this->id).'">Build log</a></span>';
 			} else {
 				throw_exception('Unrecognized build status '.$this->status);
 			}
diff --git a/shared/include/paths.php b/shared/include/paths.php
index c9f681e..1efc9d3 100644
--- a/shared/include/paths.php
+++ b/shared/include/paths.php
@@ -5,6 +5,7 @@ $paths['backend']=realpath($paths['root'].'/backend');
 $paths['frontend']=realpath($paths['root'].'/frontend');
 $paths['work']=realpath($paths['root'].'/work');
 $paths['completed']=realpath($paths['root'].'/completed');
+$paths['cache']=realpath($paths['root'].'/cache');
 foreach ($paths as $name => $dest) {
 	define(strtoupper($name), $dest);
 }
diff --git a/todo b/todo
index 8478a6e..74d6493 100644
--- a/todo
+++ b/todo
@@ -22,6 +22,4 @@ Allow backend to define bail-out functions to call when it dies (things like unm
 Add STDERR (maybe STDOUT) only option to log viewer
 *** Make frontend/backend split possible ***
 	Stop directly accessing PKGDIR from the frontend - put whatever info we need in the database and write a script for updating it from the backend
-	Have backend upload finished images to frontend
 Add options viewer - part of config process should be read-only version
-Cache the image after emerge system to save time
diff --git a/work/.gitignore b/work/.gitignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/work/.gitignore