1 # -*- coding: utf-8 -*-
5 # Tails: The Amnesic Incognito Live System
6 # Copyright © 2012 Tails developers <tails@boum.org>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 require_relative 'vagrant/lib/tails_build_settings'
29 # Path to the directory which holds our Vagrantfile
30 VAGRANT_PATH = File.expand_path('../vagrant', __FILE__)
32 # Branches that are considered 'stable' (used to select SquashFS compression)
33 STABLE_BRANCH_NAMES = ['stable', 'testing']
35 EXPORTED_VARIABLES = [
37 'TAILS_BUILD_FAILURE_RESCUE',
39 'TAILS_MERGE_BASE_BRANCH',
46 'BASE_BRANCH_GIT_COMMIT',
48 ENV['EXPORTED_VARIABLES'] = EXPORTED_VARIABLES.join(' ')
50 EXTERNAL_HTTP_PROXY = ENV['http_proxy']
53 INTERNAL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
55 ENV['ARTIFACTS'] ||= '.'
57 class CommandError < StandardError
58 attr_reader :status, :stderr
60 def initialize(message = nil, opts = {})
63 @status = opts[:status]
64 @stderr = opts[:stderr]
65 super(message % {status: @status, stderr: @stderr})
69 def run_command(*args)
70 Process.wait Kernel.spawn(*args)
72 raise CommandError.new("command #{args} failed with exit status " +
73 "%{status}", status: $?.exitstatus)
77 def capture_command(*args)
78 stdout, stderr, proc_status = Open3.capture3(*args)
79 if proc_status.exitstatus != 0
80 raise CommandError.new("command #{args} failed with exit status " +
81 "%{status}: %{stderr}",
82 stderr: stderr, status: proc_status.exitstatus)
88 question = args.first.end_with?('?')
89 args.first.sub!(/\?$/, '')
93 stdout, _ = capture_command('auto/scripts/utils.sh', *args)
94 rescue CommandError => e
104 class VagrantCommandError < CommandError
107 # Runs the vagrant command, letting stdout/stderr through. Throws an
108 # exception unless the vagrant command succeeds.
109 def run_vagrant(*args)
110 run_command('vagrant', *args, :chdir => './vagrant')
111 rescue CommandError => e
112 raise(VagrantCommandError, "'vagrant #{args}' command failed with exit " +
113 "status #{e.status}")
116 # Runs the vagrant command, not letting stdout/stderr through, and
117 # returns [stdout, stderr, Preocess:Status].
118 def capture_vagrant(*args)
119 capture_command('vagrant', *args, :chdir => './vagrant')
120 rescue CommandError => e
121 raise(VagrantCommandError, "'vagrant #{args}' command failed with exit " +
122 "status #{e.status}: #{e.stderr}")
125 [:run_vagrant, :capture_vagrant].each do |m|
126 define_method "#{m}_ssh" do |*args|
127 method(m).call('ssh', '-c', *args, '--', '-q')
131 def vagrant_ssh_config(key)
133 if $vagrant_ssh_config.nil?
134 $vagrant_ssh_config = capture_vagrant('ssh-config').first.split("\n") \
135 .map { |line| line.strip.split(/\s+/, 2) } .to_h
136 # The path in the ssh-config output is quoted, which is not what
137 # is expected outside of a shell, so let's get rid of the quotes.
138 $vagrant_ssh_config['IdentityFile'].gsub!(/^"|"$/, '')
140 $vagrant_ssh_config[key]
144 capture_vagrant_ssh('grep -c "^processor\s*:" /proc/cpuinfo').first.chomp.to_i
148 out, _ = capture_vagrant('status')
149 status_line = out.split("\n")[2]
150 if status_line['not created']
152 elsif status_line['shutoff']
154 elsif status_line['running']
157 raise "could not determine VM state"
161 def enough_free_host_memory_for_ram_build?
162 return false unless RbConfig::CONFIG['host_os'] =~ /linux/i
165 usable_free_mem = `free`.split[12].to_i
166 usable_free_mem > VM_MEMORY_FOR_RAM_BUILDS * 1024
173 capture_vagrant_ssh('free').first.chomp.split[12].to_i
176 def enough_free_vm_memory_for_ram_build?
177 free_vm_memory > BUILD_SPACE_REQUIREMENT * 1024
180 def enough_free_memory_for_ram_build?
181 if vm_state == :running
182 enough_free_vm_memory_for_ram_build?
184 enough_free_host_memory_for_ram_build?
189 git_helper('git_on_a_tag?')
193 return nil unless RbConfig::CONFIG['host_os'] =~ /linux/i
196 File.read('/proc/cpuinfo').scan(/^processor\s+:/).count
202 task :parse_build_options do
205 # Default to in-memory builds if there is enough RAM available
206 options << 'ram' if enough_free_memory_for_ram_build?
208 # Default to build using the in-VM proxy
211 # Default to fast compression on development branches
212 options << 'gzipcomp' unless is_release?
214 # Default to the number of system CPUs when we can figure it out
216 options << "cpus=#{cpus}" if cpus
218 options += ENV['TAILS_BUILD_OPTIONS'].split if ENV['TAILS_BUILD_OPTIONS']
220 options.uniq.each do |opt|
222 # Memory build settings
224 ENV['TAILS_RAM_BUILD'] = '1'
226 ENV['TAILS_RAM_BUILD'] = nil
227 # Bootstrap cache settings
228 # HTTP proxy settings
230 abort "No HTTP proxy set, but one is required by TAILS_BUILD_OPTIONS. Aborting." unless EXTERNAL_HTTP_PROXY
231 ENV['TAILS_PROXY'] = EXTERNAL_HTTP_PROXY
232 ENV['TAILS_PROXY_TYPE'] = 'extproxy'
234 ENV['TAILS_PROXY'] = INTERNAL_HTTP_PROXY
235 ENV['TAILS_PROXY_TYPE'] = 'vmproxy'
237 ENV['TAILS_PROXY'] = nil
238 ENV['TAILS_PROXY_TYPE'] = 'noproxy'
240 ENV['TAILS_OFFLINE_MODE'] = '1'
241 # SquashFS compression settings
243 ENV['MKSQUASHFS_OPTIONS'] = '-comp gzip -Xcompression-level 1'
245 raise 'We must use the default compression when building releases!'
248 ENV['MKSQUASHFS_OPTIONS'] = nil
249 # Virtual hardware settings
250 when /machinetype=([a-zA-Z0-9_.-]+)/
251 ENV['TAILS_BUILD_MACHINE_TYPE'] = $1
253 ENV['TAILS_BUILD_CPUS'] = $1
254 when /cpumodel=([a-zA-Z0-9_-]+)/
255 ENV['TAILS_BUILD_CPU_MODEL'] = $1
258 ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
259 when /dateoffset=([-+]\d+)/
260 ENV['TAILS_DATE_OFFSET'] = $1
261 # Developer convenience features
264 $force_cleanup = false
266 $force_cleanup = true
267 $keep_running = false
270 ENV['TAILS_BUILD_FAILURE_RESCUE'] = '1'
272 when 'mergebasebranch'
273 ENV['TAILS_MERGE_BASE_BRANCH'] = '1'
275 raise "Unknown Tails build option '#{opt}'"
279 if ENV['TAILS_OFFLINE_MODE'] == '1'
280 if ENV['TAILS_PROXY'].nil?
281 abort "You must use a caching proxy when building offline"
286 task :ensure_clean_repository do
287 git_status = `git status --porcelain`
288 unless git_status.empty?
289 if ENV['TAILS_BUILD_IGNORE_CHANGES']
290 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
292 You have uncommitted changes in the Git repository. They will
293 be ignored for the upcoming build:
298 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
300 You have uncommitted changes in the Git repository. Due to limitations
301 of the build system, you need to commit them before building Tails:
304 If you don't care about those changes and want to build Tails nonetheless,
305 please add `ignorechanges` to the TAILS_BUILD_OPTIONS environment
309 abort 'Uncommitted changes. Aborting.'
315 user = vagrant_ssh_config('User')
316 stdout = capture_vagrant_ssh("find '/home/#{user}/amnesia/' -maxdepth 1 " +
317 "-name 'tails-amd64-*'").first
319 rescue VagrantCommandError
324 list_artifacts.each do |artifact|
325 run_vagrant_ssh("sudo rm -f '#{artifact}'")
329 task :ensure_clean_home_directory => ['vm:up'] do
333 task :validate_http_proxy do
334 if ENV['TAILS_PROXY']
335 proxy_host = URI.parse(ENV['TAILS_PROXY']).host
338 ENV['TAILS_PROXY'] = nil
339 $stderr.puts "Ignoring invalid HTTP proxy."
343 if ['localhost', '[::1]'].include?(proxy_host) || proxy_host.start_with?('127.0.0.')
344 abort 'Using an HTTP proxy listening on the loopback is doomed to fail. Aborting.'
347 $stderr.puts "Using HTTP proxy: #{ENV['TAILS_PROXY']}"
349 $stderr.puts "No HTTP proxy set."
353 task :validate_git_state do
354 if git_helper('git_in_detached_head?') && not(git_helper('git_on_a_tag?'))
355 raise 'We are in detached head but the current commit is not tagged'
359 task :setup_environment => ['validate_git_state'] do
360 ENV['GIT_COMMIT'] ||= git_helper('git_current_commit')
361 ENV['GIT_REF'] ||= git_helper('git_current_head_name')
363 jenkins_branch = (ENV['GIT_BRANCH'] || '').sub(/^origin\//, '')
364 if not(is_release?) && jenkins_branch != ENV['GIT_REF']
365 raise "We expected to build the Git ref '#{ENV['GIT_REF']}', but GIT_REF in the environment says '#{jenkins_branch}'. Aborting!"
369 ENV['BASE_BRANCH_GIT_COMMIT'] = git_helper('git_base_branch_head')
370 ['GIT_COMMIT', 'GIT_REF', 'BASE_BRANCH_GIT_COMMIT'].each do |var|
372 raise "Variable '#{var}' is empty, which should not be possible: " +
373 "either validate_git_state is buggy or the 'origin' remote " +
374 "does not point to the official Tails Git repository."
379 task :maybe_clean_up_builder_vms do
380 clean_up_builder_vms if $force_cleanup
384 task :build => ['parse_build_options', 'ensure_clean_repository', 'maybe_clean_up_builder_vms', 'validate_git_state', 'setup_environment', 'validate_http_proxy', 'vm:up', 'ensure_clean_home_directory'] do
387 if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
388 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
390 The virtual machine is not currently set with enough memory to
391 perform an in-memory build. Either remove the `ram` option from
392 the TAILS_BUILD_OPTIONS environment variable, or shut the
393 virtual machine down using `rake vm:halt` before trying again.
396 abort 'Not enough memory for the virtual machine to run an in-memory build. Aborting.'
399 if ENV['TAILS_BUILD_CPUS'] && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i
400 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
402 The virtual machine is currently running with #{current_vm_cpus}
403 virtual CPU(s). In order to change that number, you need to
404 stop the VM first, using `rake vm:halt`. Otherwise, please
405 adjust the `cpus` options accordingly.
408 abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
411 exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
412 collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
413 run_vagrant_ssh("#{exported_env} build-tails")
415 artifacts = list_artifacts
416 raise 'No build artifacts were found!' if artifacts.empty?
417 user = vagrant_ssh_config('User')
418 hostname = vagrant_ssh_config('HostName')
419 key_file = vagrant_ssh_config('IdentityFile')
420 $stderr.puts "Retrieving artifacts from Vagrant build box."
422 "sudo chown #{user} " + artifacts.map { |a| "'#{a}'" } .join(' ')
427 # We need this since the user will not necessarily have a
428 # known_hosts entry. It is safe since an attacker must
429 # compromise libvirt's network config or the user running the
430 # command to modify the #{hostname} below.
431 '-o', 'StrictHostKeyChecking=no',
432 '-o', 'UserKnownHostsFile=/dev/null',
434 fetch_command += artifacts.map { |a| "#{user}@#{hostname}:#{a}" }
435 fetch_command << ENV['ARTIFACTS']
436 run_command(*fetch_command)
437 clean_up_builder_vms unless $keep_running
439 clean_up_builder_vms if $force_cleanup
444 not(capture_vagrant('box', 'list').grep(/^#{box_name}\s+\(libvirt,/).empty?)
448 "#{box_name}_default"
451 def clean_up_builder_vms
452 $virt = Libvirt::open("qemu:///system")
454 clean_up_domain = Proc.new do |domain|
456 domain.destroy if domain.active?
460 .lookup_storage_pool_by_name('default')
461 .lookup_volume_by_name("#{domain.name}.img")
463 rescue Libvirt::RetrieveError
464 # Expected if the pool or disk does not exist
468 # Let's ensure that the VM we are about to create is cleaned up ...
469 previous_domain = $virt.list_all_domains.find { |d| d.name == domain_name }
470 if previous_domain && previous_domain.active?
472 run_vagrant_ssh("mountpoint -q /var/cache/apt-cacher-ng")
473 rescue VagrantCommandError
474 # Nothing to unmount.
476 run_vagrant_ssh("sudo systemctl stop apt-cacher-ng.service")
477 run_vagrant_ssh("sudo umount /var/cache/apt-cacher-ng")
478 run_vagrant_ssh("sudo sync")
481 clean_up_domain.call(previous_domain)
483 # ... and the same for any residual VM based on another box (=>
484 # another domain name) that Vagrant still keeps track of.
488 open('vagrant/.vagrant/machines/default/libvirt/id', 'r') { |f| f.read }
490 $virt.lookup_domain_by_uuid(old_domain_uuid)
491 rescue Errno::ENOENT, Libvirt::RetrieveError
492 # Expected if we don't have vagrant/.vagrant, or if the VM was
493 # undefined for other reasons (e.g. manually).
496 clean_up_domain.call(old_domain)
498 # We could use `vagrant destroy` here but due to vagrant-libvirt's
499 # upstream issue #746 we then risk losing the apt-cacher-ng data.
500 # Since we essentially implement `vagrant destroy` without this bug
501 # above, but in a way so it works even if `vagrant/.vagrant` does
502 # not exist, let's just do what is safest, i.e. avoiding `vagrant
503 # destroy`. For details, see the upstream issue:
504 # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/746
505 FileUtils.rm_rf('vagrant/.vagrant')
510 desc "Remove all libvirt volumes named tails-builder-* (run at your own risk!)"
511 task :clean_up_libvirt_volumes do
512 $virt = Libvirt::open("qemu:///system")
514 pool = $virt.lookup_storage_pool_by_name('default')
515 rescue Libvirt::RetrieveError
516 # Expected if the pool does not exist
518 for disk in pool.list_volumes do
519 if /^tails-builder-/.match(disk)
521 pool.lookup_volume_by_name(disk).delete
522 rescue Libvirt::RetrieveError
523 # Expected if the disk does not exist
538 args = ARGV.drop_while { |x| x == 'test' || x == '--' }
540 args += ['--'] unless args.include? '--'
542 args += ['--tag', '~@fragile']
544 base_branch = git_helper('base_branch')
545 if git_helper('git_only_doc_changes_since?', "origin/#{base_branch}") then
546 args += ['--tag', '@doc']
549 run_command('./run_test_suite', *args)
552 desc 'Clean up all build related files'
553 task :clean_all => ['vm:destroy', 'basebox:clean_all']
556 desc 'Start the build virtual machine'
557 task :up => ['parse_build_options', 'validate_http_proxy', 'setup_environment', 'basebox:create'] do
564 rescue VagrantCommandError => e
565 clean_up_builder_vms if $force_cleanup
570 desc 'SSH into the builder VM'
575 desc 'Stop the build virtual machine'
580 desc 'Re-run virtual machine setup'
581 task :provision => ['parse_build_options', 'validate_http_proxy', 'setup_environment'] do
582 run_vagrant('provision')
585 desc "Destroy build virtual machine (clean up all files except the vmproxy's apt-cacher-ng data)"
591 namespace :basebox do
593 desc 'Create and import the base box unless already done'
596 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
598 This is the first time we are using this Vagrant base box so we
599 will have to bootstrap by building it from scratch. This will
600 take around 20 minutes (depending on your hardware) plus the
601 time needed for downloading around 250 MiB of Debian packages.
604 box_dir = VAGRANT_PATH + '/definitions/tails-builder'
605 run_command("#{box_dir}/generate-tails-builder-box.sh")
606 # Let's use an absolute path since run_vagrant changes the working
607 # directory but File.delete doesn't
608 box_path = "#{box_dir}/#{box_name}.box"
609 run_vagrant('box', 'add', '--name', box_name, box_path)
610 File.delete(box_path)
613 def basebox_date(box)
614 Date.parse(/^tails-builder-[^-]+-[^-]+-(\d{8})/.match(box)[1])
618 capture_vagrant('box', 'list').first.lines
619 .grep(/^tails-builder-.*/)
620 .map { |x| x.chomp.sub(/\s.*$/, '') }
623 def clean_up_basebox(box)
624 run_vagrant('box', 'remove', '--force', box)
626 $virt = Libvirt::open("qemu:///system")
628 .lookup_storage_pool_by_name('default')
629 .lookup_volume_by_name("#{box}_vagrant_box_image_0.img")
631 rescue Libvirt::RetrieveError
632 # Expected if the pool or disk does not exist
638 desc 'Remove all base boxes'
640 baseboxes.each { |box| clean_up_basebox(box) }
643 desc 'Remove all base boxes older than six months'
646 # We always want to keep the newest basebox
647 boxes.sort! { |a, b| basebox_date(a) <=> basebox_date(b) }
650 if basebox_date(box) < Date.today - 365.0/2.0
651 clean_up_basebox(box)