[rubygems/rubygems] Use a constant empty tar header to avoid extra allocations
[ruby.git] / tool / make-snapshot
blob7446f18578da830341f7085a435d43b094106d0f
1 #!/usr/bin/ruby -s
2 # -*- coding: us-ascii -*-
3 require 'rubygems'
4 require 'rubygems/package'
5 require 'rubygems/package/tar_writer'
6 require 'uri'
7 require 'digest/sha1'
8 require 'digest/sha2'
9 require 'fileutils'
10 require 'shellwords'
11 require 'tmpdir'
12 require 'pathname'
13 require 'yaml'
14 require 'json'
15 require File.expand_path("../lib/vcs", __FILE__)
16 require File.expand_path("../lib/colorize", __FILE__)
17 STDOUT.sync = true
19 $srcdir ||= nil
20 $archname = nil if ($archname ||= nil) == ""
21 $keep_temp ||= nil
22 $patch_file ||= nil
23 $packages ||= nil
24 $digests ||= nil
25 $no7z ||= nil
26 $tooldir = File.expand_path("..", __FILE__)
27 $unicode_version = nil if ($unicode_version ||= nil) == ""
28 $colorize = Colorize.new
30 def usage
31 <<USAGE
32 usage: #{File.basename $0} [option...] new-directory-to-save [version ...]
33 options:
34 -srcdir=PATH source directory path
35 -archname=NAME make the basename of snapshots NAME
36 -keep_temp keep temporary working directory
37 -patch_file=PATCH apply PATCH file after export
38 -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")})
39 -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")})
40 -unicode_version=VER Unicode version to generate encodings
41 -help, --help show this message
42 version:
43 master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL
44 each versions may be followed by optional @revision.
45 USAGE
46 end
48 DIGESTS = %w[SHA1 SHA256 SHA512]
49 PACKAGES = {
50 "tar" => %w".tar",
51 "bzip" => %w".tar.bz2 bzip2 -c",
52 "gzip" => %w".tar.gz gzip -c",
53 "xz" => %w".tar.xz xz -c",
54 "zip" => %w".zip zip -Xqr",
56 DEFAULT_PACKAGES = PACKAGES.keys - ["tar"]
57 if !$no7z and system("7z", out: IO::NULL)
58 PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so"
59 PACKAGES["zip"] = %w".zip 7z a -tzip -l -mx -mtc=off" << {out: IO::NULL}
60 elsif gzip = ENV.delete("GZIP")
61 PACKAGES["gzip"].concat(gzip.shellsplit)
62 end
64 if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags
65 mflags = mflags.gsub(/(\A|\s)(-\S*)j\d*/, '\1\2')
66 mflags.strip!
67 ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?)
68 end
69 ENV["LC_ALL"] = ENV["LANG"] = "C"
70 # https git clone is disabled at git.ruby-lang.org/ruby.git.
71 GITURL = URI.parse("https://github.com/ruby/ruby.git")
72 RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/
74 ENV["VPATH"] ||= "include/ruby"
75 YACC = ENV["YACC"] ||= "#{$tooldir}/lrama/exe/lrama"
76 ENV["BASERUBY"] ||= "ruby"
77 ENV["RUBY"] ||= "ruby"
78 ENV["MV"] ||= "mv"
79 ENV["RM"] ||= "rm -f"
80 ENV["MINIRUBY"] ||= "ruby"
81 ENV["PROGRAM"] ||= "ruby"
82 ENV["AUTOCONF"] ||= "autoconf"
83 ENV["BUILTIN_TRANSOBJS"] ||= "newline.o"
84 ENV["TZ"] = "UTC"
86 class String
87 # for older ruby
88 alias bytesize size unless method_defined?(:bytesize)
89 end
91 class Dir
92 def self.mktmpdir(path)
93 path = File.join(tmpdir, path+"-#{$$}-#{rand(100000)}")
94 begin
95 mkdir(path)
96 rescue Errno::EEXIST
97 path.succ!
98 retry
99 end
100 path
101 end unless respond_to?(:mktmpdir)
104 $packages &&= $packages.split(/[, ]+/).tap {|pkg|
105 if all = pkg.index("all")
106 pkg[all, 1] = DEFAULT_PACKAGES - pkg
108 pkg -= PACKAGES.keys
109 pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}"
111 $packages ||= DEFAULT_PACKAGES
113 $digests &&= $digests.split(/[, ]+/).tap {|dig|
114 dig -= DIGESTS
115 dig.empty? or abort "#{File.basename $0}: unknown digests - #{dig.join(", ")}"
117 $digests ||= DIGESTS
119 $patch_file &&= File.expand_path($patch_file)
120 path = ENV["PATH"].split(File::PATH_SEPARATOR)
121 %w[YACC BASERUBY RUBY MV MINIRUBY].each do |var|
122 cmd, = ENV[var].shellsplit
123 unless path.any? {|dir|
124 file = File.expand_path(cmd, dir)
125 File.file?(file) and File.executable?(file)
127 abort "#{File.basename $0}: #{var} command not found - #{cmd}"
131 %w[BASERUBY RUBY MINIRUBY].each do |var|
132 %x[#{ENV[var]} --disable-gem -e1 2>&1]
133 if $?.success?
134 ENV[var] += ' --disable-gem'
138 if defined?($help) or defined?($_help)
139 puts usage
140 exit
142 unless destdir = ARGV.shift
143 abort usage
145 revisions = ARGV.empty? ? [nil] : ARGV
147 if defined?($exported)
148 abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead"
151 FileUtils.mkpath(destdir)
152 destdir = File.expand_path(destdir)
153 tmp = Dir.mktmpdir("ruby-snapshot")
154 FileUtils.mkpath(tmp)
155 at_exit {
156 Dir.chdir "/"
157 FileUtils.rm_rf(tmp)
158 } unless $keep_temp
160 def tar_create(tarball, dir)
161 header = Gem::Package::TarHeader
162 dir_type = "5"
163 uname = gname = "ruby"
164 File.open(tarball, "wb") do |f|
165 w = Gem::Package::TarWriter.new(f)
166 list = Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH)
167 list.reject! {|name| name.end_with?("/.")}
168 list.sort_by! {|name| name.split("/")}
169 list.each do |path|
170 next if File.basename(path) == "."
171 s = File.stat(path)
172 mode = 0644
173 case
174 when s.file?
175 type = nil
176 size = s.size
177 mode |= 0111 if s.executable?
178 when s.directory?
179 path += "/"
180 type = dir_type
181 size = 0
182 mode |= 0111
183 else
184 next
186 name, prefix = w.split_name(path)
187 h = header.new(name: name, prefix: prefix, typeflag: type,
188 mode: mode, size: size, mtime: s.mtime,
189 uname: uname, gname: gname)
190 f.write(h)
191 if size > 0
192 IO.copy_stream(path, f)
193 f.write("\0" * (-size % 512))
197 true
198 rescue => e
199 warn e.message
200 false
203 def touch_all(time, pattern, opt, &cond)
204 Dir.glob(pattern, opt) do |n|
205 stat = File.stat(n)
206 if stat.file? or stat.directory?
207 next if cond and !yield(n, stat)
208 File.utime(time, time, n)
211 rescue
212 false
213 else
214 true
217 class MAKE < Struct.new(:prog, :args)
218 def initialize(vars)
219 vars = vars.map {|arg| arg.join("=")}
220 super(ENV["MAKE"] || ENV["make"] || "make", vars)
223 def run(target)
224 err = IO.pipe do |r, w|
225 begin
226 pid = Process.spawn(self.prog, *self.args, target, {:err => w, r => :close})
227 w.close
228 r.read
229 ensure
230 Process.wait(pid)
233 if $?.success?
234 true
235 else
236 STDERR.puts err
237 $colorize.fail("#{target} failed")
238 false
243 def measure
244 clock = Process::CLOCK_MONOTONIC
245 t0 = Process.clock_gettime(clock)
246 STDOUT.flush
247 result = yield
248 printf(" %6.3f", Process.clock_gettime(clock) - t0)
249 STDOUT.flush
250 result
253 def package(vcs, rev, destdir, tmp = nil)
254 pwd = Dir.pwd
255 patchlevel = false
256 prerelease = false
257 if rev and revision = rev[/@(\h+)\z/, 1]
258 rev = $`
260 case rev
261 when nil
262 url = nil
263 when /\A(?:master|trunk)\z/
264 url = vcs.trunk
265 when /\Abranches\//
266 url = vcs.branch($')
267 when /\Atags\//
268 url = vcs.tag($')
269 when /\Astable\z/
270 vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]}
271 url &&= vcs.branch(url)
272 when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/
273 prerelease = true
274 tag = "#{$4}#{$5}"
275 url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}")
276 when /\A(.*)\.(.*)\.(.*)-p(\d+)/
277 patchlevel = true
278 tag = "p#{$4}"
279 url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}")
280 when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/
281 if $3 && ($1 > "2" || $1 == "2" && $2 >= "1")
282 patchlevel = true
283 tag = ""
284 url = vcs.tag("v#{$1}_#{$2}_#{$3}")
285 else
286 url = vcs.branch("ruby_#{rev.tr('.', '_')}")
288 else
289 warn "#{$0}: unknown version - #{rev}"
290 return
292 if info = vcs.get_revisions(url)
293 modified = info[2]
294 else
295 _, _, modified = VCS::Null.new(nil).get_revisions(url)
297 if !revision and info
298 revision = info
299 url ||= vcs.branch(revision[3])
300 revision = revision[1]
302 version = nil
303 unless revision
304 url = vcs.trunk
305 vcs.grep(RUBY_VERSION_PATTERN, url, "version.h") {version = $1}
306 unless rev == version
307 warn "#{$0}: #{rev} not found"
308 return
310 revision = vcs.get_revisions(url)[1]
313 v = "ruby"
314 puts "Exporting #{rev}@#{revision}"
315 exported = tmp ? File.join(tmp, v) : v
316 unless vcs.export(revision, url, exported, true) {|line| print line}
317 warn("Export failed")
318 return
320 if $srcdir
321 Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file|
322 puts "copying #{file}" if $VERBOSE
323 dest = exported + file[$srcdir.size..-1]
324 FileUtils.mkpath(File.dirname(dest))
325 begin
326 FileUtils.cp_r(file, dest)
327 FileUtils.chmod_R("a+rwX,go-w", dest)
328 rescue SystemCallError
333 Dir.glob("#{exported}/.*.yml") do |file|
334 FileUtils.rm(file, verbose: $VERBOSE)
337 status = IO.read(File.dirname(__FILE__) + "/prereq.status")
338 Dir.chdir(tmp) if tmp
340 if !File.directory?(v)
341 v = Dir.glob("ruby-*").select(&File.method(:directory?))
342 v.size == 1 or abort "#{File.basename $0}: not exported"
343 v = v[0]
346 File.open("#{v}/revision.h", "wb") {|f|
347 f.puts vcs.revision_header(revision, modified)
349 version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1]
350 version ||=
351 begin
352 include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h")
353 api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1]
354 api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1]
355 version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1]
356 [api_major_version, api_minor_version, version_teeny].join('.')
358 version or return
359 if patchlevel
360 unless tag.empty?
361 versionhdr ||= IO.read("#{v}/version.h")
362 patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1]
363 tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision))
365 elsif prerelease
366 versionhdr ||= IO.read("#{v}/version.h")
367 versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace"
368 IO.write("#{v}/version.h", versionhdr)
369 else
370 tag ||= vcs.revision_name(revision)
373 if $archname
374 n = $archname
375 elsif tag.empty?
376 n = "ruby-#{version}"
377 else
378 n = "ruby-#{version}-#{tag}"
380 File.directory?(n) or File.rename v, n
381 v = n
383 if $patch_file && !system(*%W"patch -d #{v} -p0 -i #{$patch_file}")
384 puts $colorize.fail("patching failed")
385 return
388 class << (clean = [])
389 def add(n) push(n)
392 def create(file, content = "", &block)
393 add(file)
394 if block
395 File.open(file, "wb", &block)
396 else
397 File.binwrite(file, content)
402 Dir.chdir(v) do
403 unless File.exist?("ChangeLog")
404 vcs.export_changelog(url, nil, revision, "ChangeLog")
407 unless touch_all(modified, "**/*", File::FNM_DOTMATCH)
408 modified = nil
409 colors = %w[red yellow green cyan blue magenta]
410 "take a breath, and go ahead".scan(/./) do |c|
411 if c == ' '
412 print c
413 else
414 colors.push(color = colors.shift)
415 print $colorize.decorate(c, color)
417 sleep(c == "," ? 0.7 : 0.05)
419 puts
422 clean.create("cross.rb") do |f|
423 f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)"
424 f.puts "CROSS_COMPILING=true"
425 f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)"
426 f.puts "RUBY_PLATFORM='none'"
427 f.puts "Object.__send__(:remove_const, :RUBY_VERSION)"
428 f.puts "RUBY_VERSION='#{version}'"
430 puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE
431 unless File.exist?("configure")
432 print "creating configure..."
433 unless system([ENV["AUTOCONF"]]*2)
434 puts $colorize.fail(" failed")
435 return
437 puts $colorize.pass(" done")
439 clean.add("autom4te.cache")
440 clean.add("enc/unicode/data")
441 print "creating prerequisites..."
442 if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk")
443 puts
444 extout = clean.add('tmp')
445 begin
446 status = IO.read("tool/prereq.status")
447 rescue Errno::ENOENT
448 # use fallback file
450 clean.create("config.status", status)
451 clean.create("noarch-fake.rb", "require_relative 'cross'\n")
452 FileUtils.mkpath(hdrdir = "#{extout}/include/ruby")
453 File.binwrite("#{hdrdir}/config.h", "")
454 FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults")
455 File.binwrite("#{defaults}/operating_system.rb", "")
456 File.binwrite("#{defaults}/ruby.rb", "")
457 miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross"
458 baseruby = ENV["BASERUBY"]
459 mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")).
460 gsub(/^@.*\n/, '')
461 vars = {
462 "EXTOUT"=>extout,
463 "PATH_SEPARATOR"=>File::PATH_SEPARATOR,
464 "MINIRUBY"=>miniruby,
465 "RUBY"=>ENV["RUBY"],
466 "BASERUBY"=>baseruby,
467 "PWD"=>Dir.pwd,
468 "ruby_version"=>version,
469 "MAJOR"=>api_major_version,
470 "MINOR"=>api_minor_version,
471 "TEENY"=>version_teeny,
473 status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do
474 vars[$2] ||= $3
476 vars.delete("UNICODE_FILES") # for stable branches
477 vars["UNICODE_VERSION"] = $unicode_version if $unicode_version
478 args = vars.dup
479 mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]}
480 mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile')
481 mk << <<-'APPEND'
483 update-download:: touch-unicode-files
484 prepare-package: prereq after-update
485 clean-cache: $(CLEAN_CACHE)
486 after-update:: extract-gems
487 extract-gems: update-gems
488 update-gems:
489 $(UNICODE_SRC_DATA_DIR)/.unicode-tables.time:
490 touch-unicode-files:
491 APPEND
492 clean.create("Makefile", mk)
493 clean.create("revision.tmp")
494 clean.create(".revision.time")
495 ENV["CACHE_SAVE"] = "no"
496 make = MAKE.new(args)
497 return unless make.run("update-download")
498 clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk", "ext/ripper/y.output", ".revision.time")
499 Dir.glob("**/*") do |dest|
500 next unless File.symlink?(dest)
501 orig = File.expand_path(File.readlink(dest), File.dirname(dest))
502 File.unlink(dest)
503 FileUtils.cp_r(orig, dest)
505 File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"]))
506 return unless make.run("prepare-package")
507 return unless make.run("clean-cache")
508 if modified
509 new_time = modified + 2
510 touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat|
511 stat.mtime > modified unless clean.include?(name)
513 modified = new_time
515 print "prerequisites"
516 else
517 system(*%W"#{YACC} -o parse.c parse.y")
519 vcs.after_export(".") if exported
520 clean.concat(Dir.glob("ext/**/autom4te.cache"))
521 clean.add(".downloaded-cache")
522 if File.exist?("gems/bundled_gems")
523 gems = Dir.glob("gems/*.gem")
524 gems -= File.readlines("gems/bundled_gems").map {|line|
525 next if /^\s*(?:#|$)/ =~ line
526 name, version, _ = line.split(' ')
527 "gems/#{name}-#{version}.gem"
529 clean.concat(gems)
530 else
531 clean.add("gems")
533 FileUtils.rm_rf(clean)
534 if modified
535 touch_all(modified, "**/*/", 0) do |name, stat|
536 stat.mtime > modified
538 File.utime(modified, modified, ".")
540 unless $?.success?
541 puts $colorize.fail(" failed")
542 return
544 puts $colorize.pass(" done")
547 if v == "."
548 v = File.basename(Dir.pwd)
549 Dir.chdir ".."
550 else
551 Dir.chdir(File.dirname(v))
552 v = File.basename(v)
555 tarball = nil
556 return $packages.collect do |mesg|
557 (ext, *cmd) = PACKAGES[mesg]
558 File.directory?(destdir) or FileUtils.mkpath(destdir)
559 file = File.join(destdir, "#{$archname||v}#{ext}")
560 case ext
561 when /\.tar/
562 if tarball
563 next if tarball.empty?
564 else
565 tarball = ext == ".tar" ? file : "#{$archname||v}.tar"
566 print "creating tarball... #{tarball}"
567 if measure {tar_create(tarball, v)}
568 puts $colorize.pass(" done")
569 File.utime(modified, modified, tarball) if modified
570 next if tarball == file
571 else
572 puts $colorize.fail(" failed")
573 tarball = ""
574 next
577 print "creating #{mesg} tarball... #{file}"
578 done = measure {system(*cmd, tarball, out: file)}
579 else
580 print "creating #{mesg} archive... #{file}"
581 if Hash === cmd.last
582 *cmd, opt = *cmd
583 cmd << file << v << opt
584 else
585 (cmd = cmd.dup) << file << v
587 done = measure {system(*cmd)}
589 if done
590 puts $colorize.pass(" done")
591 file
592 else
593 puts $colorize.fail(" failed")
596 end.compact
597 ensure
598 FileUtils.rm_rf(tmp ? File.join(tmp, v) : v) if v and !$keep_temp
599 Dir.chdir(pwd)
602 if [$srcdir, ($git||=nil)].compact.size > 1
603 abort "#{File.basename $0}: -srcdir and -git are exclusive"
605 if $srcdir
606 vcs = VCS.detect($srcdir)
607 elsif $git
608 abort "#{File.basename $0}: use -srcdir with cloned local repository"
609 else
610 begin
611 vcs = VCS.detect(File.expand_path("../..", __FILE__))
612 rescue VCS::NotFoundError
613 abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository"
617 release_date = Time.now.getutc
618 info = {}
620 success = true
621 revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name|
622 if !name
623 success = false
624 next
626 str = File.binread(name)
627 pathname = Pathname(name)
628 basename = pathname.basename.to_s
629 extname = pathname.extname.sub(/\A\./, '')
630 version = basename[/\Aruby-(.*)\.(?:tar|zip)/, 1]
631 key = basename[/\A(.*)\.(?:tar|zip)/, 1]
632 info[key] ||= Hash.new{|h,k|h[k]={}}
633 info[key]['version'] = version if version
634 info[key]['date'] = release_date.strftime('%Y-%m-%d')
635 if version
636 info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/"
637 info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}"
638 else
639 info[key]['filename'][extname] = basename
641 info[key]['size'][extname] = str.bytesize
642 puts "* #{$colorize.pass(name)}"
643 puts " SIZE: #{str.bytesize} bytes"
644 $digests.each do |alg|
645 digest = Digest(alg).hexdigest(str)
646 info[key][alg.downcase][extname] = digest
647 printf " %-8s%s\n", "#{alg}:", digest
651 yaml = info.values.to_yaml
652 json = info.values.to_json
653 puts "#{$colorize.pass('YAML:')}"
654 puts yaml
655 puts "#{$colorize.pass('JSON:')}"
656 puts json
657 infodir = Pathname(destdir) + 'info'
658 infodir.mkpath
659 (infodir+'info.yml').write(yaml)
660 (infodir+'info.json').write(json)
662 exit false if !success
664 # vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix