3 # tool/update-deps verify makefile dependencies.
6 # gcc 4.5 (for -save-temps=obj option)
7 # GNU make (for -p option)
9 # Warning: ccache (and similar tools) must be disabled for
10 # -save-temps=obj to work properly.
13 # 1. Compile ruby with -save-temps=obj option.
14 # Ex. ./configure debugflags='-save-temps=obj -g' && make all golf
15 # 2. Run tool/update-deps to show dependency problems.
16 # Ex. ./ruby tool/update-deps
17 # 3. Use --fix to fix makefiles.
18 # Ex. ./ruby tool/update-deps --fix
21 # * Fix makefiles using previously detected dependency problems
22 # Ex. ruby tool/update-deps --actual-fix [file]
23 # "ruby tool/update-deps --fix" is the same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix".
31 # When out-of-place build, files may be built in source directory or
33 # Some files are always built in the source directory.
34 # Some files are always built in the build directory.
35 # Some files are built in the source directory for tarball but build directory for repository (svn).
38 How to build test directories.
42 tar xf ruby-$VER-r$REV.tar.xz
43 cp -a ruby-$VER-r$REV tarball_source_dir_original
44 mv ruby-$VER-r$REV tarball_source_dir_after_build
45 svn co -q -r$REV https://svn.ruby-lang.org/repos/ruby/trunk ruby
47 cp -a ruby repo_source_dir_original
48 mv ruby repo_source_dir_after_build
49 mkdir tarball_build_dir repo_build_dir tarball_install_dir repo_install_dir
50 (cd tarball_build_dir; ../tarball_source_dir_after_build/configure --prefix=$(cd ../tarball_install_dir; pwd) && make all golf install) > tarball.log 2>&1
51 (cd repo_build_dir; ../repo_source_dir_after_build/configure --prefix=$(cd ../repo_install_dir; pwd) && make all golf install) > repo.log 2>&1
54 repo_source_dir_original
55 repo_source_dir_after_build
57 tarball_source_dir_original
58 tarball_source_dir_after_build
64 Dir.chdir(d) { Find.find(".") {|f| files[d][f] = true if %r{\.(c|h|inc|dmyh)\z} =~ f } }
67 files_union = files.values.map {|h| h.keys }.flatten.uniq.sort
69 k = files.map {|d,h| h[f] ? d : nil }.compact.sort
70 next if k == %w[repo_source_dir_after_build repo_source_dir_original tarball_source_dir_after_build tarball_source_dir_original]
71 next if k == %w[repo_build_dir tarball_build_dir] && File.basename(f) == "extconf.h"
80 puts " " + f.sub(%r{\A\./}, "")
87 # Files built in the source directory.
88 # They can be referenced as $(top_srcdir)/filename.
89 # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)'
90 FILES_IN_SOURCE_DIRECTORY
= %w
[
93 # Files built in the build directory (except extconf.h).
94 # They can be referenced as $(topdir)/filename.
95 # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts(g("tarball_build_dir").reject {|f| %r{/extconf.h\z} =~ f }.sort)'
96 FILES_IN_BUILD_DIRECTORY
= %w
[
99 ext
/socket/constdefs
.c
100 ext
/socket/constdefs
.h
106 # They are built in the build directory if the source is obtained from the repository.
107 # However they are pre-built for tarball and they exist in the source directory extracted from the tarball.
108 # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_build_dir") & g("tarball_source_dir_original")).sort)'
109 FILES_NEED_VPATH
= %w
[
110 ext
/rbconfig/sizeof
/sizes
.c
111 ext
/ripper/eventids1
.c
112 ext
/ripper/eventids2table
.c
114 ext
/ripper/ripper_init
.c
137 enc
/trans/emoji_iso2022_kddi
.c
138 enc
/trans/emoji_sjis_docomo
.c
139 enc
/trans/emoji_sjis_kddi
.c
140 enc
/trans/emoji_sjis_softbank
.c
146 enc
/trans/japanese_euc
.c
147 enc
/trans/japanese_sjis
.c
149 enc
/trans/single_byte
.c
151 enc
/trans/utf_16_32
.c
164 # Multiple files with same filename.
165 # It is not good idea to refer them using VPATH.
166 # Files in FILES_SAME_NAME_INC is referenced using $(hdrdir).
167 # Files in FILES_SAME_NAME_TOP is referenced using $(top_srcdir).
169 FILES_SAME_NAME_INC
= %w
[
172 include/ruby/version.h
175 FILES_SAME_NAME_TOP
= %w
[
179 # Files that may or may not exist on CI for some reason.
180 # Windows build generally seems to have missing dependencies.
181 UNSTABLE_FILES
= %r
{\Awin32
/[^/]+\
.o\z
}
183 # Other source files exist in the source directory.
185 def in_makefile(target
, source
)
189 when %r
{\A
[^
/]*\z}, %r{\Acoroutine/}, %r
{\Aprism
/}
190 target2
= "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
192 when *FILES_IN_SOURCE_DIRECTORY
then source2
= "$(top_srcdir)/#{source}"
193 when *FILES_IN_BUILD_DIRECTORY
then source2
= "{$(VPATH)}#{source}" # VPATH is not used now but it may changed in future.
194 when *FILES_NEED_VPATH
then source2
= "{$(VPATH)}#{source}"
195 when *FILES_SAME_NAME_INC
then source2
= "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}"
196 when *FILES_SAME_NAME_TOP
then source2
= "$(top_srcdir)/#{source}"
197 when 'thread_pthread.c' then source2
= '{$(VPATH)}thread_$(THREAD_MODEL).c'
198 when 'thread_pthread.h' then source2
= '{$(VPATH)}thread_$(THREAD_MODEL).h'
199 when %r
{\A
[^
/]*\z
} then source2
= "{$(VPATH)}#{File.basename source}"
200 when %r
{\A\
.ext
/include/[^
/]+/ruby
/} then source2
= "{$(VPATH)}#{$'}"
201 when %r
{\Ainclude
/ruby/} then source2
= "{$(VPATH)}#{$'}"
202 when %r
{\Aenc
/} then source2
= "{$(VPATH)}#{$'}"
203 when %r
{\Amissing
/} then source2
= "{$(VPATH)}#{$'}"
204 when %r
{\Accan
/} then source2 = "$(CCAN_DIR)/#{$'}"
205 when %r
{\Adefs
/} then source2
= "{$(VPATH)}#{source}"
206 when %r
{\Acoroutine
/} then source2
= "{$(VPATH)}$(COROUTINE_H)"
207 else source2
= "$(top_srcdir)/#{source}"
209 ["common.mk", target2
, source2
]
211 target2
= "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
213 when *FILES_IN_SOURCE_DIRECTORY
then source2
= "$(top_srcdir)/#{source}"
214 when *FILES_IN_BUILD_DIRECTORY
then source2
= source
215 when *FILES_NEED_VPATH
then source2
= source
216 when *FILES_SAME_NAME_INC
then source2
= "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}"
217 when *FILES_SAME_NAME_TOP
then source2
= "$(top_srcdir)/#{source}"
218 when %r
{\A\
.ext
/include/[^
/]+/ruby
/} then source2
= $
'
219 when %r{\Ainclude/ruby/} then source2 = $'
220 when %r
{\Aenc
/unicode/[\d
.]+/} then source2 = '$(UNICODE_HDR_DIR)/' + $'
221 when %r
{\Aenc
/} then source2
= source
222 else source2
= "$(top_srcdir)/#{source}"
224 ["enc/depend", target2
, source2
]
226 targetdir
= File
.dirname(target
)
227 unless File
.exist
?("#{targetdir}/extconf.rb")
228 warn
"warning: not found: #{targetdir}/extconf.rb"
230 target2
= File
.basename(target
)
231 relpath
= Pathname(source
).relative_path_from(Pathname(target
).dirname
).to_s
233 when *FILES_IN_SOURCE_DIRECTORY
then source2
= "$(top_srcdir)/#{source}"
234 when *FILES_IN_BUILD_DIRECTORY
then source2
= relpath
235 when *FILES_NEED_VPATH
then source2
= "{$(VPATH)}#{File.basename source}"
236 when *FILES_SAME_NAME_INC
then source2
= "$(hdrdir)/#{source.sub(%r{\Ainclude/},'')}"
237 when *FILES_SAME_NAME_TOP
then source2
= "$(top_srcdir)/#{source}"
238 when %r
{\A\
.ext
/include/[^
/]+/ruby
/} then source2 = "$(arch_hdrdir)/ruby
/#{$'}"
239 when %r
{\Ainclude
/} then source2 = "$(hdrdir)/#{$'}"
240 when %r
{\A
#{Regexp.escape targetdir}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)"
241 when %r
{\A
#{Regexp.escape targetdir}/} then source2 = $'
242 when %r
{\A
#{Regexp.escape File.dirname(targetdir)}/} then source2 = "$(srcdir)/../#{$'}"
243 else source2
= "$(top_srcdir)/#{source}"
245 ["#{File.dirname(target)}/depend", target2
, source2
]
246 # Files that may or may not exist on CI for some reason.
247 # Windows build generally seems to have missing dependencies.
249 warn
"warning: ignoring: #{target}"
251 raise "unexpected target: #{target}"
255 DEPENDENCIES_SECTION_START_MARK
= "\# AUTOGENERATED DEPENDENCIES START\n"
256 DEPENDENCIES_SECTION_END_MARK
= "\# AUTOGENERATED DEPENDENCIES END\n"
260 if mkflag0
= ENV['GNUMAKEFLAGS'] and (mkflag
= mkflag0
.sub(/(\A|\s+)-j\d*(?=\s+|\z)/, '')) != mkflag0
262 ENV['GNUMAKEFLAGS'] = mkflag
267 $opt_actual_fix = false
272 op
= OptionParser
.new
273 op
.banner
= 'Usage: ruby tool/update-deps'
274 op
.def_option('-a', 'show valid dependencies') { $opt_a = true }
275 op
.def_option('--fix') { $opt_fix = true }
276 op
.def_option('--actual-fix') { $opt_actual_fix = true }
280 def read_make_deps(cwd
)
282 make_p
, make_p_stderr
, make_p_status
= Open3
.capture3("make -p all miniruby exe/ruby golf")
283 File
.open('update-deps.make.out.log', 'w') {|f
| f
.print make_p
}
284 File
.open('update-deps.make.err.log', 'w') {|f
| f
.print make_p_stderr
}
285 if !make_p_status
.success
?
291 make_p
.scan(%r
{Entering\ directory\
['`](.*)'|
294 ^
([/0-9a-zA-Z
._-
]+):(.*)\n((?:\
#.*\n)*)|
295 ^\
#\ (Finished\ Make\ data\ base\ on)\ |
296 Leaving\ directory\
['`](.*)'}x
) {
299 data_base_curdir
= $3
307 enter_dir
= Pathname(directory_enter
)
308 #p [:enter, enter_dir]
309 dirstack
.push enter_dir
310 elsif data_base_start
312 elsif data_base_curdir
313 curdir
= Pathname(data_base_curdir
)
314 elsif rule_target
&& rule_sources
&& rule_desc
&&
315 /Modification time never checked/ !~ rule_desc
# This pattern match eliminates rules which VPATH is not expanded.
318 deps
= deps
.scan(%r
{[/0-9a-zA-Z
._-
]+})
319 deps
.delete_if
{|dep
| /\.time\z/ =~ dep
} # skip timestamp
320 next if /\.o\z/ !~ target
.to_s
321 next if /libyjit.o\z/ =~ target
.to_s
# skip YJIT Rust object (no corresponding C source)
322 next if /\.bundle\// =~ target
.to_s
323 next if /\A\./ =~ target
.to_s
# skip rules such as ".c.o"
324 #p [curdir, target, deps]
325 dir
= curdir
|| dirstack
.last
326 dependencies
[dir
+ target
] ||= []
327 dependencies
[dir
+ target
] |= deps
.map
{|dep
| dir
+ dep
}
330 elsif directory_leave
331 leave_dir
= Pathname(directory_leave
)
332 #p [:leave, leave_dir]
333 if leave_dir
!= dirstack
.last
334 warn
"unexpected leave_dir : #{dirstack.last.inspect} != #{leave_dir.inspect}"
342 #def guess_compiler_wd(filename, hint0)
345 # guess = hint + filename
350 # end while hint.to_s != '.'
351 # raise ArgumentError, "can not find #{filename} (hint: #{hint0})"
354 def read_single_cc_deps(path_i
, cwd
, fn_o
)
357 path_i
.each_line
{|line
|
358 next if /\A\# \d+ "(.*)"/ !~ line
361 next if %r
{\A
<.*>\z
} =~ dep
# omit <command-line>, etc.
362 next if /\.e?rb\z/ =~ dep
363 # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line.
365 compiler_wd
= Pathname(dep
.sub(%r
{//\z
}, ''))
371 compiler_wd
||= fn_o
.to_s
.start_with
?("enc/") ? cwd
: path_i
.parent
374 files
.each_key
{|dep
|
377 dep
= compiler_wd
+ dep
380 warn
"warning: file not found: #{dep}"
383 next if !dep
.to_s
.start_with
?(cwd
.to_s
) # omit system headers.
386 if deps
.include?(cwd
+ "probes.h")
387 deps
<< (cwd
+ "probes.dmyh")
392 def read_cc_deps(cwd
)
394 Pathname
.glob('**/*.o').sort
.each
{|fn_o
|
395 fn_i
= fn_o
.sub_ext('.i')
397 next if fn_o
.sub_ext('.S').exist
?
398 warn
"warning: not found: #{fn_i}"
404 deps
[path_o
] = read_single_cc_deps(path_i
, cwd
, fn_o
)
409 def concentrate(dependencies
, cwd
)
411 dependencies
.keys
.sort
.each
{|target
|
412 sources
= dependencies
[target
]
413 target
= target
.relative_path_from(cwd
)
414 sources
= sources
.map
{|s
|
415 rel
= s
.relative_path_from(cwd
)
418 if %r
{\A\
.\
.(/|\z
)} =~ target
.to_s
419 warn
"warning: out of tree target: #{target}"
422 sources
= sources
.reject
{|s
|
423 if %r
{\A\
.\
.(/|\z
)} =~ s
.to_s
424 warn
"warning: out of tree source: #{s}"
426 elsif %r
{/\
.time\z
} =~ s
.to_s
432 deps
[target
] = sources
437 def sort_paths(paths
)
439 ary
= t
.to_s
.split(%r
{/})
440 ary
.map
.with_index
{|e
, i
| i
== ary
.length-1
? [0, e
] : [1, e
] } # regular file first, directories last.
444 def show_deps(tag
, deps
)
445 targets
= sort_paths(deps
.keys
)
447 sources
= sort_paths(deps
[t
])
449 puts
"#{tag} #{t}: #{s}"
454 def detect_dependencies(out
=$stdout)
456 make_deps
= read_make_deps(cwd
)
458 make_deps
= concentrate(make_deps
, cwd
)
460 cc_deps
= read_cc_deps(cwd
)
462 cc_deps
= concentrate(cc_deps
, cwd
)
464 return make_deps
, cc_deps
467 def compare_deps(make_deps
, cc_deps
, out
=$stdout)
468 targets
= make_deps
.keys
| cc_deps
.keys
473 make_deps
.each
{|t
, sources
|
475 makefile
, t2
, s2
= in_makefile(t
, s
)
476 makefiles
[makefile
] = true
477 make_lines_hash
[makefile
] ||= Hash
.new(false)
478 make_lines_hash
[makefile
]["#{t2}: #{s2}"] = true
483 cc_deps
.each
{|t
, sources
|
485 makefile
, t2
, s2
= in_makefile(t
, s
)
486 makefiles
[makefile
] = true
487 cc_lines_hash
[makefile
] ||= Hash
.new(false)
488 cc_lines_hash
[makefile
]["#{t2}: #{s2}"] = true
492 makefiles
.keys
.compact
.sort
.each
{|makefile
|
493 cc_lines
= cc_lines_hash
[makefile
] || Hash
.new(false)
494 make_lines
= make_lines_hash
[makefile
] || Hash
.new(false)
500 if /^
#{Regexp.escape DEPENDENCIES_SECTION_START_MARK}
502 #{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/x =~ content
503 pre_post_part
= [$
`, $']
504 current_lines = Hash.new(false)
505 $1.each_line {|line| current_lines[line.chomp] = true }
506 (cc_lines.keys | current_lines.keys | make_lines.keys).sort.each {|line|
507 status = [cc_lines[line], current_lines[line], make_lines[line]]
509 when [true, true, true]
511 when [true, true, false]
512 out.puts "warning #{makefile} : #{line} (make doesn't detect written dependency)"
513 when [true, false, true]
514 out.puts "add_auto #{makefile} : #{line} (harmless)" # This is automatically updatable.
515 when [true, false, false]
516 out.puts "add_auto #{makefile} : #{line} (harmful)" # This is automatically updatable.
517 when [false, true, true]
518 out.puts "del_cc #{makefile} : #{line}" # Not automatically updatable because build on other OS may need the dependency.
519 when [false, true, false]
520 out.puts "del_cc #{makefile} : #{line} (Curious. make doesn't detect this dependency.)" # Not automatically updatable because build on other OS may need the dependency.
521 when [false, false, true]
522 out.puts "del_make #{makefile} : #{line}" # Not automatically updatable because the dependency is written manually.
524 raise "unexpected status: #{status.inspect}"
528 (cc_lines.keys | make_lines.keys).sort.each {|line|
529 status = [cc_lines[line], make_lines[line]]
534 out.puts "add_manual #{makefile} : #{line}" # Not automatically updatable because makefile has no section to update automatically.
536 out.puts "del_manual #{makefile} : #{line}" # Not automatically updatable because makefile has no section to update automatically.
538 raise "unexpected status: #{status.inspect}"
546 unless File.exist?("Makefile")
547 if File.exist?("autogen.sh")
548 system("./autogen.sh")
549 elsif !File.exist?("configure")
550 system("autoreconf", "-i", "-s")
552 system("./configure", "-q", "--enable-load-relative", "--prefix=/.",
553 "--disable-install-doc", "debugflags=-save-temps=obj -g")
557 def main_show(out=$stdout)
559 make_deps, cc_deps = detect_dependencies(out)
560 compare_deps(make_deps, cc_deps, out)
563 def extract_deplines(problems)
566 problems.each_line {|line|
568 when /\Aadd_auto (\S+) : ((\S+): (\S+))/
569 (adds[$1] ||= []) << [line, "#{$2}\n"]
570 when /\A(?:del_cc|del_make|add_manual|del_manual|warning) (\S+) : /
571 (others[$1] ||= []) << line
573 raise "unexpected line: #{line.inspect}"
579 def main_actual_fix(problems)
580 adds, others = extract_deplines(problems)
581 (adds.keys | others.keys).sort.each {|makefile|
589 /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK}
591 #{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/x =~ content
592 pre_dep_post = [$`, $1, $
']
597 if pre_dep_post && adds[makefile]
598 pre_lines, dep_lines, post_lines = pre_dep_post
599 dep_lines = dep_lines.lines.to_a
600 add_lines = adds[makefile].map(&:last)
601 new_lines = (dep_lines | add_lines).sort.uniq
604 DEPENDENCIES_SECTION_START_MARK,
606 DEPENDENCIES_SECTION_END_MARK,
609 if content != new_content
610 puts "modified: #{makefile}"
611 tmp_makefile = "#{makefile}.new.#{$$}"
612 File.write(tmp_makefile, new_content)
613 File.rename tmp_makefile, makefile
614 (add_lines - dep_lines).each {|line| puts " added #{line}" }
616 puts "not modified: #{makefile}"
619 others[makefile].each {|line| puts " #{line}" }
623 puts "no additional lines: #{makefile}"
625 puts "no dependencies section: #{makefile}"
627 puts "no makefile: #{makefile}"
630 puts " warning: dependencies section was exist at previous phase."
633 adds[makefile].map(&:first).each {|line| puts " #{line}" }
636 others[makefile].each {|line| puts " #{line}" }
643 problems = StringIO.new
645 main_actual_fix(problems.string)
652 main_actual_fix(ARGF.read)
663 warn "warning: missing *.i files, see help in #$0 and ensure ccache is disabled"