Merge pull request #12 from p8952/docker

Replace Vagrant with Docker 

14 files changed, 115 insertions, 57 deletions

diff --git a/web/Gemfile b/web/Gemfile
index b86a535..61aa55d 100644
--- a/web/Gemfile
+++ b/web/Gemfile
@@ -1,5 +1,7 @@
 source 'https://rubygems.org'
 
+gem 'archive-tar-minitar'
+gem 'docker-api'
 gem 'gems'
 gem 'net-scp'
 gem 'net-ssh'
@@ -8,7 +10,6 @@ gem 'pmap'
 gem 'puma'
 gem 'sequel'
 gem 'sinatra'
-gem 'vagrant_rbapi'
 
 group :development do
 	gem 'minitest'
diff --git a/web/Gemfile.lock b/web/Gemfile.lock
index d009a6d..66308a8 100644
--- a/web/Gemfile.lock
+++ b/web/Gemfile.lock
@@ -1,14 +1,21 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    archive-tar-minitar (0.5.2)
     ast (2.0.0)
     astrolabe (1.3.0)
       parser (>= 2.2.0.pre.3, < 3.0)
     celluloid (0.16.0)
       timers (~> 4.0.0)
+    docker-api (1.17.0)
+      archive-tar-minitar
+      excon (>= 0.38.0)
+      json
+    excon (0.44.1)
     ffi (1.9.6)
     gems (0.8.3)
     hitimes (1.2.2)
+    json (1.8.2)
     listen (2.8.5)
       celluloid (>= 0.15.2)
       rb-fsevent (>= 0.9.3)
@@ -30,7 +37,6 @@ GEM
     rack-test (0.6.3)
       rack (>= 1.0)
     rainbow (2.0.0)
-    rake (10.4.2)
     rb-fsevent (0.9.4)
     rb-inotify (0.9.5)
       ffi (>= 0.5.0)
@@ -51,15 +57,13 @@ GEM
     tilt (1.4.1)
     timers (4.0.1)
       hitimes
-    vagrant_rbapi (0.0.2)
-      net-scp
-      net-ssh
-      rake
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  archive-tar-minitar
+  docker-api
   gems
   minitest
   net-scp
@@ -72,4 +76,3 @@ DEPENDENCIES
   rubocop
   sequel
   sinatra
-  vagrant_rbapi
diff --git a/web/Rakefile b/web/Rakefile
index 3f3dfab..86aa390 100644
--- a/web/Rakefile
+++ b/web/Rakefile
@@ -1,10 +1,23 @@
 require_relative 'app'
 
 task default: 'test'
+
+desc 'Run the test suite'
 task :test do
 	Dir.glob('./test/test_*.rb') { |f| require f }
 end
 
+desc 'Run : update_packages => run_ci_untested => update_ci => run_repoman => update_repoman'
+task :nightly do
+	Rake::Task['db:update_packages'].invoke
+	Rake::Task['docker:setup'].invoke
+	Rake::Task['docker:run_ci_untested'].invoke
+	Rake::Task['db:update_ci'].invoke
+	Rake::Task['docker:run_repoman'].invoke
+	Rake::Task['db:update_repoman'].invoke
+	Rake::Task['docker:teardown'].invoke
+end
+
 namespace :db do
 	DB.loggers << Logger.new($stdout)
 
@@ -13,49 +26,71 @@ namespace :db do
 		Sequel::Migrator.run(DB, 'db/migrations')
 	end
 
+	desc 'Update the packages database with new versions and targets'
 	task :update_packages do
 		update_packages
 	end
 
+	desc 'Update the build database with logfiles from ci-logs/'
 	task :update_ci do
 		update_ci
 	end
 
+	desc 'Clear the build database'
 	task :clear_ci do
 		clear_ci
 	end
 
+	desc 'Update the repoman database with logfiles from repo-logs/'
 	task :update_repoman do
 		update_repoman
 	end
 
-	task :tidy_repoman do
-		tidy_repoman
-	end
-
+	desc 'Clear the repoman database'
 	task :clear_repoman do
 		clear_repoman
 	end
 end
 
-namespace :vm do
-	provisioner = ENV['provisioner']
-	provisioner = 'virtualbox' if provisioner.nil?
+namespace :docker do
+	num_of_packages = ENV['NUM_OF_PACKAGES'].to_i
+	num_of_packages = 5 if num_of_packages == 0
 
+	desc 'Build a docker image to use with subsequent tasks'
+	task :setup do
+		Docker.options[:read_timeout] = 36_000
+		Docker.options[:write_timeout] = 36_000
+		docker_path = File.dirname(File.expand_path(File.dirname(__FILE__)))
+		@docker_image = Docker::Image.build_from_dir(docker_path)
+	end
+
+	desc 'Remove a previously built docker image'
+	task :teardown do
+		@docker_image.remove
+	end
+
+	desc 'Build and test all packages'
 	task :run_ci_all do
-		run_ci(:all, provisioner)
+		run_ci(@docker_image, :all)
 	end
 
-	num_of_packages = ENV['num_of_packages'].to_i
+	desc 'Build and test a fixed number of packages (NUM_OF_PACKAGES=5)'
 	task :run_ci_some do
-		run_ci(num_of_packages, provisioner)
+		run_ci(@docker_image, num_of_packages)
 	end
 
+	desc 'Build and test all untested packages and their reverse dependencies'
 	task :run_ci_untested do
-		run_ci(:untested, provisioner)
+		run_ci(@docker_image, :untested)
+	end
+
+	desc 'QA test all packages'
+	task :run_repoman_all do
+		run_repoman(@docker_image, :all)
 	end
 
-	task :run_repoman do
-		run_repoman(provisioner)
+	desc 'QA test a fixed number of packages (NUM_OF_PACKAGES=5)'
+	task :run_repoman_some do
+		run_repoman(@docker_image, num_of_packages)
 	end
 end
diff --git a/web/app.rb b/web/app.rb
index 8377feb..0a9d735 100644
--- a/web/app.rb
+++ b/web/app.rb
@@ -1,3 +1,5 @@
+require 'archive/tar/minitar'
+require 'docker'
 require 'gems'
 require 'logger'
 require 'net/scp'
@@ -5,7 +7,6 @@ require 'net/ssh'
 require 'pmap'
 require 'sequel'
 require 'sinatra/base'
-require 'vagrant_rbapi'
 
 require_relative 'lib/ci'
 require_relative 'lib/helpers'
diff --git a/web/ci-logs/.gitignore b/web/ci-logs/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/web/ci-logs/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/web/ci-logs/.keep b/web/ci-logs/.keep
deleted file mode 100644
index e69de29..0000000
--- a/web/ci-logs/.keep
+++ /dev/null
diff --git a/web/lib/ci.rb b/web/lib/ci.rb
index 64d649e..300ba53 100644
--- a/web/lib/ci.rb
+++ b/web/lib/ci.rb
@@ -1,4 +1,4 @@
-def run_ci(num_of_packages, provisioner)
+def run_ci(docker_image, num_of_packages)
 	packages = []
 	Package.order { [category, lower(name), version] }.each do |package|
 		packages << package[:identifier]
@@ -6,8 +6,6 @@ def run_ci(num_of_packages, provisioner)
 
 	if num_of_packages == :all
 		packages = packages
-	elsif num_of_packages == 0
-		packages = packages.sample(5)
 	elsif num_of_packages == :untested
 		packages = []
 		Package.exclude(tested: true).order { [category, lower(name), version] }.each do |package|
@@ -32,17 +30,22 @@ def run_ci(num_of_packages, provisioner)
 		packages = packages.sample(num_of_packages)
 	end
 
-	exit if packages.empty?
+	packages = packages.uniq
+	packages.each do |package|
+		docker_container = docker_image.run("/ruby-tinderbox/tinder.sh #{package}")
+		docker_container.wait(36_000)
 
-	begin
-		vagrant_path = File.dirname(File.dirname(File.expand_path(File.dirname(__FILE__))))
-		vagrant = Vagrant_Rbapi.new(vagrant_path)
-		vagrant.up(provisioner)
-		sleep 5 while vagrant.status != 'running'
-		vagrant.ssh('sudo /vagrant/tinder.sh ' + packages.join(' '))
-		vagrant.scp(:download, true, '/vagrant/ci-logs', 'web')
-	ensure
-		vagrant.destroy
+		tar = Tempfile.new('tar')
+		File.open(tar, 'w') do |file|
+			docker_container.copy('/ruby-tinderbox/ci-logs') do |chunk|
+				file.write(chunk)
+			end
+		end
+		Archive::Tar::Minitar.unpack(tar, File.dirname(File.expand_path(File.dirname(__FILE__))))
+		tar.close
+		tar.unlink
+
+		docker_container.delete
 	end
 end
 
diff --git a/web/lib/models.rb b/web/lib/models.rb
index ff0b3ee..667e54f 100644
--- a/web/lib/models.rb
+++ b/web/lib/models.rb
@@ -1,4 +1,4 @@
-DB = Sequel.connect(ENV['DATABASE_URL'], max_connections: 12 , pool_timeout: 60)
+DB = Sequel.connect(ENV['DATABASE_URL'], max_connections: 12, pool_timeout: 60)
 
 class Package < Sequel::Model
 end
diff --git a/web/lib/repoman.rb b/web/lib/repoman.rb
index 8287837..6808340 100644
--- a/web/lib/repoman.rb
+++ b/web/lib/repoman.rb
@@ -1,4 +1,4 @@
-def run_repoman(provisioner)
+def run_repoman(docker_image, num_of_packages)
 	packages = []
 	Package.order { [category, lower(name), version] }.each do |package|
 		target = ''
@@ -6,19 +6,13 @@ def run_repoman(provisioner)
 		target = package[:r20_target] unless package[:r20_target] == 'nil'
 		target = package[:r21_target] unless package[:r21_target] == 'nil'
 		target = package[:r22_target] unless package[:r22_target] == 'nil'
-		if target.empty?
-			Repoman.where(package_id: package[:identifier]).delete
-			next
-		end
+		next if target.empty?
 
 		next_target = ''
 		next_target = 'ruby20' if target == 'ruby19'
 		next_target = 'ruby21' if target == 'ruby20'
 		next_target = 'ruby22' if target == 'ruby21'
-		if next_target.empty?
-			Repoman.where(package_id: package[:identifier]).delete
-			next
-		end
+		next if next_target.empty?
 
 		category = package[:category]
 		name = package[:name]
@@ -28,18 +22,33 @@ def run_repoman(provisioner)
 		packages << "#{category} #{name} #{version}#{revision} #{target} #{next_target}"
 	end
 
-	packages = "'" + packages.join("' '") + "'"
-
-	begin
-		vagrant_path = File.dirname(File.dirname(File.expand_path(File.dirname(__FILE__))))
-		vagrant = Vagrant_Rbapi.new(vagrant_path)
-		vagrant.up(provisioner)
-		sleep 5 while vagrant.status != 'running'
-		vagrant.ssh('sudo /vagrant/repoman.sh ' + packages)
-		vagrant.scp(:download, true, '/vagrant/repo-logs', 'web')
-	ensure
-		vagrant.destroy
+	if num_of_packages == :all
+		packages = packages
+	else
+		packages = packages.sample(num_of_packages)
 	end
+
+	packages = packages.uniq
+	packages.each do |package|
+		package = "'" + package + "'"
+	end
+	packages = packages.unshift('/ruby-tinderbox/repoman.sh')
+
+	docker_container = docker_image.run(packages)
+	docker_container.wait(36_000)
+
+	tar = Tempfile.new('tar')
+	File.open(tar, 'w') do |file|
+		docker_container.copy('/ruby-tinderbox/repo-logs') do |chunk|
+			file.write(chunk)
+			puts chunk
+		end
+	end
+	Archive::Tar::Minitar.unpack(tar, File.dirname(File.expand_path(File.dirname(__FILE__))))
+	tar.close
+	tar.unlink
+
+	docker_container.delete
 end
 
 def update_repoman
@@ -78,9 +87,7 @@ def update_repoman
 			next_log: next_log
 		)
 	end
-end
 
-def tidy_repoman
 	Package.order { [category, lower(name), version] }.each do |package|
 		target = ''
 		target = package[:r19_target] unless package[:r19_target] == 'nil'
diff --git a/web/repo-logs/.gitignore b/web/repo-logs/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/web/repo-logs/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/web/var/log/.gitignore b/web/var/log/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/web/var/log/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/web/var/log/.keep b/web/var/log/.keep
deleted file mode 100644
index e69de29..0000000
--- a/web/var/log/.keep
+++ /dev/null
diff --git a/web/var/run/.gitignore b/web/var/run/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/web/var/run/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/web/var/run/.keep b/web/var/run/.keep
deleted file mode 100644
index e69de29..0000000
--- a/web/var/run/.keep
+++ /dev/null