1 # -*- coding: utf-8 -*-
5 # Tails: The Amnesic Incognito Live System
6 # Copyright © 2012 Tails developers <tails@boum.org>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 require_relative 'vagrant/lib/tails_build_settings'
29 # Path to the directory which holds our Vagrantfile
30 VAGRANT_PATH = File.expand_path('../vagrant', __FILE__)
32 # Branches that are considered 'stable' (used to select SquashFS compression)
33 STABLE_BRANCH_NAMES = ['stable', 'testing']
35 EXPORTED_VARIABLES = [
37 'APT_SNAPSHOTS_SERIALS',
38 'TAILS_BUILD_FAILURE_RESCUE',
40 'TAILS_MERGE_BASE_BRANCH',
47 'BASE_BRANCH_GIT_COMMIT',
49 ENV['EXPORTED_VARIABLES'] = EXPORTED_VARIABLES.join(' ')
51 EXTERNAL_HTTP_PROXY = ENV['http_proxy']
54 INTERNAL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
56 ENV['ARTIFACTS'] ||= '.'
58 ENV['APT_SNAPSHOTS_SERIALS'] ||= ''
60 class CommandError < StandardError
61 attr_reader :status, :stderr
63 def initialize(message = nil, opts = {})
66 @status = opts[:status]
67 @stderr = opts[:stderr]
68 super(message % {status: @status, stderr: @stderr})
72 def run_command(*args)
73 Process.wait Kernel.spawn(*args)
75 raise CommandError.new("command #{args} failed with exit status " +
76 "%{status}", status: $?.exitstatus)
80 def capture_command(*args)
81 stdout, stderr, proc_status = Open3.capture3(*args)
82 if proc_status.exitstatus != 0
83 raise CommandError.new("command #{args} failed with exit status " +
84 "%{status}: %{stderr}",
85 stderr: stderr, status: proc_status.exitstatus)
91 question = args.first.end_with?('?')
92 args.first.sub!(/\?$/, '')
96 stdout, _ = capture_command('auto/scripts/utils.sh', *args)
97 rescue CommandError => e
107 class VagrantCommandError < CommandError
110 # Runs the vagrant command, letting stdout/stderr through. Throws an
111 # exception unless the vagrant command succeeds.
112 def run_vagrant(*args)
113 run_command('vagrant', *args, :chdir => './vagrant')
114 rescue CommandError => e
115 raise(VagrantCommandError, "'vagrant #{args}' command failed with exit " +
116 "status #{e.status}")
119 # Runs the vagrant command, not letting stdout/stderr through, and
120 # returns [stdout, stderr, Preocess:Status].
121 def capture_vagrant(*args)
122 capture_command('vagrant', *args, :chdir => './vagrant')
123 rescue CommandError => e
124 raise(VagrantCommandError, "'vagrant #{args}' command failed with exit " +
125 "status #{e.status}: #{e.stderr}")
128 [:run_vagrant, :capture_vagrant].each do |m|
129 define_method "#{m}_ssh" do |*args|
130 method(m).call('ssh', '-c', *args, '--', '-q')
134 def vagrant_ssh_config(key)
136 if $vagrant_ssh_config.nil?
137 $vagrant_ssh_config = capture_vagrant('ssh-config').first.split("\n") \
138 .map { |line| line.strip.split(/\s+/, 2) } .to_h
139 # The path in the ssh-config output is quoted, which is not what
140 # is expected outside of a shell, so let's get rid of the quotes.
141 $vagrant_ssh_config['IdentityFile'].gsub!(/^"|"$/, '')
143 $vagrant_ssh_config[key]
147 capture_vagrant_ssh('grep -c "^processor\s*:" /proc/cpuinfo').first.chomp.to_i
151 out, _ = capture_vagrant('status')
152 status_line = out.split("\n")[2]
153 if status_line['not created']
155 elsif status_line['shutoff']
157 elsif status_line['running']
160 raise "could not determine VM state"
164 def enough_free_host_memory_for_ram_build?
165 return false unless RbConfig::CONFIG['host_os'] =~ /linux/i
168 usable_free_mem = `free`.split[12].to_i
169 usable_free_mem > VM_MEMORY_FOR_RAM_BUILDS * 1024
176 capture_vagrant_ssh('free').first.chomp.split[12].to_i
179 def enough_free_vm_memory_for_ram_build?
180 free_vm_memory > BUILD_SPACE_REQUIREMENT * 1024
183 def enough_free_memory_for_ram_build?
184 if vm_state == :running
185 enough_free_vm_memory_for_ram_build?
187 enough_free_host_memory_for_ram_build?
192 git_helper('git_on_a_tag?')
196 return nil unless RbConfig::CONFIG['host_os'] =~ /linux/i
199 File.read('/proc/cpuinfo').scan(/^processor\s+:/).count
205 task :parse_build_options do
208 # Default to in-memory builds if there is enough RAM available
209 options << 'ram' if enough_free_memory_for_ram_build?
211 # Default to build using the in-VM proxy
214 # Default to fast compression on development branches
215 options << 'gzipcomp' unless is_release?
217 # Default to the number of system CPUs when we can figure it out
219 options << "cpus=#{cpus}" if cpus
221 options += ENV['TAILS_BUILD_OPTIONS'].split if ENV['TAILS_BUILD_OPTIONS']
223 options.uniq.each do |opt|
225 # Memory build settings
227 ENV['TAILS_RAM_BUILD'] = '1'
229 ENV['TAILS_RAM_BUILD'] = nil
230 # Bootstrap cache settings
231 # HTTP proxy settings
233 abort "No HTTP proxy set, but one is required by TAILS_BUILD_OPTIONS. Aborting." unless EXTERNAL_HTTP_PROXY
234 ENV['TAILS_PROXY'] = EXTERNAL_HTTP_PROXY
235 ENV['TAILS_PROXY_TYPE'] = 'extproxy'
237 ENV['TAILS_PROXY'] = INTERNAL_HTTP_PROXY
238 ENV['TAILS_PROXY_TYPE'] = 'vmproxy'
240 ENV['TAILS_PROXY'] = nil
241 ENV['TAILS_PROXY_TYPE'] = 'noproxy'
243 ENV['TAILS_OFFLINE_MODE'] = '1'
244 # SquashFS compression settings
246 ENV['MKSQUASHFS_OPTIONS'] = '-comp gzip -Xcompression-level 1'
248 raise 'We must use the default compression when building releases!'
251 ENV['MKSQUASHFS_OPTIONS'] = nil
252 # Virtual hardware settings
253 when /machinetype=([a-zA-Z0-9_.-]+)/
254 ENV['TAILS_BUILD_MACHINE_TYPE'] = $1
256 ENV['TAILS_BUILD_CPUS'] = $1
257 when /cpumodel=([a-zA-Z0-9_-]+)/
258 ENV['TAILS_BUILD_CPU_MODEL'] = $1
261 ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
262 when /dateoffset=([-+]\d+)/
263 ENV['TAILS_DATE_OFFSET'] = $1
264 # Developer convenience features
267 $force_cleanup = false
269 $force_cleanup = true
270 $keep_running = false
273 ENV['TAILS_BUILD_FAILURE_RESCUE'] = '1'
275 when 'mergebasebranch'
276 ENV['TAILS_MERGE_BASE_BRANCH'] = '1'
278 raise "Unknown Tails build option '#{opt}'"
282 if ENV['TAILS_OFFLINE_MODE'] == '1'
283 if ENV['TAILS_PROXY'].nil?
284 abort "You must use a caching proxy when building offline"
289 task :ensure_clean_repository do
290 git_status = `git status --porcelain`
291 unless git_status.empty?
292 if ENV['TAILS_BUILD_IGNORE_CHANGES']
293 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
295 You have uncommitted changes in the Git repository. They will
296 be ignored for the upcoming build:
301 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
303 You have uncommitted changes in the Git repository. Due to limitations
304 of the build system, you need to commit them before building Tails:
307 If you don't care about those changes and want to build Tails nonetheless,
308 please add `ignorechanges` to the TAILS_BUILD_OPTIONS environment
312 abort 'Uncommitted changes. Aborting.'
318 user = vagrant_ssh_config('User')
319 stdout = capture_vagrant_ssh("find '/home/#{user}/amnesia/' -maxdepth 1 " +
320 "-name 'tails-amd64-*' " +
321 "-o -name tails-build-env.list").first
323 rescue VagrantCommandError
328 list_artifacts.each do |artifact|
329 run_vagrant_ssh("sudo rm -f '#{artifact}'")
333 task :ensure_clean_home_directory => ['vm:up'] do
337 task :validate_http_proxy do
338 if ENV['TAILS_PROXY']
339 proxy_host = URI.parse(ENV['TAILS_PROXY']).host
342 ENV['TAILS_PROXY'] = nil
343 $stderr.puts "Ignoring invalid HTTP proxy."
347 if ['localhost', '[::1]'].include?(proxy_host) || proxy_host.start_with?('127.0.0.')
348 abort 'Using an HTTP proxy listening on the loopback is doomed to fail. Aborting.'
351 $stderr.puts "Using HTTP proxy: #{ENV['TAILS_PROXY']}"
353 $stderr.puts "No HTTP proxy set."
357 task :validate_git_state do
358 if git_helper('git_in_detached_head?') && not(git_helper('git_on_a_tag?'))
359 raise 'We are in detached head but the current commit is not tagged'
363 task :setup_environment => ['validate_git_state'] do
364 ENV['GIT_COMMIT'] ||= git_helper('git_current_commit')
365 ENV['GIT_REF'] ||= git_helper('git_current_head_name')
367 jenkins_branch = (ENV['GIT_BRANCH'] || '').sub(/^origin\//, '')
368 if not(is_release?) && jenkins_branch != ENV['GIT_REF']
369 raise "We expected to build the Git ref '#{ENV['GIT_REF']}', but GIT_REF in the environment says '#{jenkins_branch}'. Aborting!"
373 ENV['BASE_BRANCH_GIT_COMMIT'] = git_helper('git_base_branch_head')
374 ['GIT_COMMIT', 'GIT_REF', 'BASE_BRANCH_GIT_COMMIT'].each do |var|
376 raise "Variable '#{var}' is empty, which should not be possible: " +
377 "either validate_git_state is buggy or the 'origin' remote " +
378 "does not point to the official Tails Git repository."
383 task :maybe_clean_up_builder_vms do
384 clean_up_builder_vms if $force_cleanup
388 task :build => ['parse_build_options', 'ensure_clean_repository', 'maybe_clean_up_builder_vms', 'validate_git_state', 'setup_environment', 'validate_http_proxy', 'vm:up', 'ensure_clean_home_directory'] do
391 if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
392 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
394 The virtual machine is not currently set with enough memory to
395 perform an in-memory build. Either remove the `ram` option from
396 the TAILS_BUILD_OPTIONS environment variable, or shut the
397 virtual machine down using `rake vm:halt` before trying again.
400 abort 'Not enough memory for the virtual machine to run an in-memory build. Aborting.'
403 if ENV['TAILS_BUILD_CPUS'] && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i
404 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
406 The virtual machine is currently running with #{current_vm_cpus}
407 virtual CPU(s). In order to change that number, you need to
408 stop the VM first, using `rake vm:halt`. Otherwise, please
409 adjust the `cpus` options accordingly.
412 abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
415 exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
416 collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
417 run_vagrant_ssh("#{exported_env} build-tails")
419 artifacts = list_artifacts
420 raise 'No build artifacts were found!' if artifacts.empty?
421 user = vagrant_ssh_config('User')
422 hostname = vagrant_ssh_config('HostName')
423 key_file = vagrant_ssh_config('IdentityFile')
424 $stderr.puts "Retrieving artifacts from Vagrant build box."
426 "sudo chown #{user} " + artifacts.map { |a| "'#{a}'" } .join(' ')
431 # We need this since the user will not necessarily have a
432 # known_hosts entry. It is safe since an attacker must
433 # compromise libvirt's network config or the user running the
434 # command to modify the #{hostname} below.
435 '-o', 'StrictHostKeyChecking=no',
436 '-o', 'UserKnownHostsFile=/dev/null',
438 fetch_command += artifacts.map { |a| "#{user}@#{hostname}:#{a}" }
439 fetch_command << ENV['ARTIFACTS']
440 run_command(*fetch_command)
441 clean_up_builder_vms unless $keep_running
443 clean_up_builder_vms if $force_cleanup
448 not(capture_vagrant('box', 'list').grep(/^#{box_name}\s+\(libvirt,/).empty?)
452 "#{box_name}_default"
455 def clean_up_builder_vms
456 $virt = Libvirt::open("qemu:///system")
458 clean_up_domain = Proc.new do |domain|
460 domain.destroy if domain.active?
464 .lookup_storage_pool_by_name('default')
465 .lookup_volume_by_name("#{domain.name}.img")
467 rescue Libvirt::RetrieveError
468 # Expected if the pool or disk does not exist
472 # Let's ensure that the VM we are about to create is cleaned up ...
473 previous_domain = $virt.list_all_domains.find { |d| d.name == domain_name }
474 if previous_domain && previous_domain.active?
476 run_vagrant_ssh("mountpoint -q /var/cache/apt-cacher-ng")
477 rescue VagrantCommandError
478 # Nothing to unmount.
480 run_vagrant_ssh("sudo systemctl stop apt-cacher-ng.service")
481 run_vagrant_ssh("sudo umount /var/cache/apt-cacher-ng")
482 run_vagrant_ssh("sudo sync")
485 clean_up_domain.call(previous_domain)
487 # ... and the same for any residual VM based on another box (=>
488 # another domain name) that Vagrant still keeps track of.
492 open('vagrant/.vagrant/machines/default/libvirt/id', 'r') { |f| f.read }
494 $virt.lookup_domain_by_uuid(old_domain_uuid)
495 rescue Errno::ENOENT, Libvirt::RetrieveError
496 # Expected if we don't have vagrant/.vagrant, or if the VM was
497 # undefined for other reasons (e.g. manually).
500 clean_up_domain.call(old_domain)
502 # We could use `vagrant destroy` here but due to vagrant-libvirt's
503 # upstream issue #746 we then risk losing the apt-cacher-ng data.
504 # Since we essentially implement `vagrant destroy` without this bug
505 # above, but in a way so it works even if `vagrant/.vagrant` does
506 # not exist, let's just do what is safest, i.e. avoiding `vagrant
507 # destroy`. For details, see the upstream issue:
508 # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/746
509 FileUtils.rm_rf('vagrant/.vagrant')
514 desc "Remove all libvirt volumes named tails-builder-* (run at your own risk!)"
515 task :clean_up_libvirt_volumes do
516 $virt = Libvirt::open("qemu:///system")
518 pool = $virt.lookup_storage_pool_by_name('default')
519 rescue Libvirt::RetrieveError
520 # Expected if the pool does not exist
522 for disk in pool.list_volumes do
523 if /^tails-builder-/.match(disk)
525 pool.lookup_volume_by_name(disk).delete
526 rescue Libvirt::RetrieveError
527 # Expected if the disk does not exist
542 args = ARGV.drop_while { |x| x == 'test' || x == '--' }
544 args += ['--'] unless args.include? '--'
546 args += ['--tag', '~@fragile']
548 base_branch = git_helper('base_branch')
549 if git_helper('git_only_doc_changes_since?', "origin/#{base_branch}") then
550 args += ['--tag', '@doc']
553 run_command('./run_test_suite', *args)
556 desc 'Clean up all build related files'
557 task :clean_all => ['vm:destroy', 'basebox:clean_all']
560 desc 'Start the build virtual machine'
561 task :up => ['parse_build_options', 'validate_http_proxy', 'setup_environment', 'basebox:create'] do
568 rescue VagrantCommandError => e
569 clean_up_builder_vms if $force_cleanup
574 desc 'SSH into the builder VM'
579 desc 'Stop the build virtual machine'
584 desc 'Re-run virtual machine setup'
585 task :provision => ['parse_build_options', 'validate_http_proxy', 'setup_environment'] do
586 run_vagrant('provision')
589 desc "Destroy build virtual machine (clean up all files except the vmproxy's apt-cacher-ng data)"
595 namespace :basebox do
597 desc 'Create and import the base box unless already done'
600 $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
602 This is the first time we are using this Vagrant base box so we
603 will have to bootstrap by building it from scratch. This will
604 take around 20 minutes (depending on your hardware) plus the
605 time needed for downloading around 250 MiB of Debian packages.
608 box_dir = VAGRANT_PATH + '/definitions/tails-builder'
609 run_command("#{box_dir}/generate-tails-builder-box.sh")
610 # Let's use an absolute path since run_vagrant changes the working
611 # directory but File.delete doesn't
612 box_path = "#{box_dir}/#{box_name}.box"
613 run_vagrant('box', 'add', '--name', box_name, box_path)
614 File.delete(box_path)
617 def basebox_date(box)
618 Date.parse(/^tails-builder-[^-]+-[^-]+-(\d{8})/.match(box)[1])
622 capture_vagrant('box', 'list').first.lines
623 .grep(/^tails-builder-.*/)
624 .map { |x| x.chomp.sub(/\s.*$/, '') }
627 def clean_up_basebox(box)
628 run_vagrant('box', 'remove', '--force', box)
630 $virt = Libvirt::open("qemu:///system")
632 .lookup_storage_pool_by_name('default')
633 .lookup_volume_by_name("#{box}_vagrant_box_image_0.img")
635 rescue Libvirt::RetrieveError
636 # Expected if the pool or disk does not exist
642 desc 'Remove all base boxes'
644 baseboxes.each { |box| clean_up_basebox(box) }
647 desc 'Remove all base boxes older than six months'
650 # We always want to keep the newest basebox
651 boxes.sort! { |a, b| basebox_date(a) <=> basebox_date(b) }
654 if basebox_date(box) < Date.today - 365.0/2.0
655 clean_up_basebox(box)