1 ##===- llvm-test/Makefile.programs -------------------------*- Makefile -*-===##
3 # This makefile contains all of the makefile machinery that is common to
4 # building stuff in this directory. This script can be used in two
5 # different ways. The main job of this is to take executables for the following
8 # 1. The native platform compiler
9 # 2. LLVM Bytecode Compiler + LLI interpreter (if ENABLE_LLI is enabled)
10 # 3. LLVM Bytecode Compiler + LLC static compiler
11 # 4. LLVM Bytecode Compiler + C Backend + Native Sun Compiler
12 # 5. LLVM Bytecode Compiler + LLI Just-In-Time Compiler
14 # Running them, and then diffing the output. If there are any failures, they
15 # are flagged. The other mode is used in conjunction with the TEST=<testname>
16 # argument on the make command line. In this case, a Makefile.TEST.<testname>
17 # makefile is used to run tests on the program (see below).
19 # Input to this makefile should be the PROGRAMS_TO_TEST variable, which contains
20 # a list of programs that should be run. The makefile can also optionally
21 # specify a STDIN_FILENAME variable, which contains a filename that is piped
22 # into the program as it is being executed.
24 # FIXME: There should be a way to specify the command line for a program
26 ##===----------------------------------------------------------------------===##
28 # Running Custom Tests
30 # This makefile provides facilities for defining custom tests that are to be run
31 # on all of the programs in the test suite. In order to define one of these
32 # tests, create a llvm-test/TEST.<testname>.Makefile file. This file
33 # should define a rule 'test.<testname>.%' which is run once for each program in
34 # the suite (the % passed in is the program name). For a simple example, see
35 # Makefile.TEST.example.
37 ##===----------------------------------------------------------------------===##
39 # Dependencies on header files need to be determined explicitly because
40 # we do not automatically compute dependencies
41 INCLUDES := $(ExtraHeaders) $(wildcard $(SourceDir)/*.h)
43 include $(LEVEL)/Makefile.tests
45 .PRECIOUS: Output/%.llvm Output/%.native Output/%.simple Output/%.llc Output/%.llc.s
46 .PRECIOUS: Output/%.cbe Output/%.cbe.c Output/%.llvm.bc Output/%.linked.bc
47 .PRECIOUS: Output/%.linked.optbeta.bc Output/%.llvm.optbeta.bc
49 PROGDIR = $(PROJ_SRC_ROOT)
52 # Scripts in the this directory...
55 # TIMEPROG - The program used to get timing results for a program
56 TIMEPROG := $(PROGDIR)/TimeProgram.sh
60 TOLERANCEOPT += -r $(FP_TOLERANCE)
63 TOLERANCEOPT += -a $(FP_ABSTOLERANCE)
66 # DIFFPROG - The program used to diff the output
68 # We run this under RunToolSafely because 'fpcmp' at one point would infinite
69 # loop on some inputs, which blocks testing some historical revisions. We can
70 # remove this once we build our own test tools.
71 DIFFPROG := $(PROGDIR)/RunToolSafely.sh 60 \
72 $(PROGDIR)/DiffOutput.sh "\"$(FPCMP) $(TOLERANCEOPT)\""
74 # RUNTIMELIMIT - The number of seconds we should wait before certain events
75 # timeout. This is overridable on the commandline or in tests makefiles.
81 # If the program specified a REFERENCE_OUTPUT_FILE, they obviously want to
82 # USE_REFERENCE_OUTPUT.
83 ifdef REFERENCE_OUTPUT_FILE
84 USE_REFERENCE_OUTPUT := 1
87 # Figure out what kind of configuration specific reference output file to look
89 ifdef SMALL_PROBLEM_SIZE
90 REFERENCE_OUTPUT_KEY := small
92 ifdef LARGE_PROBLEM_SIZE
93 REFERENCE_OUTPUT_KEY := large
95 REFERENCE_OUTPUT_KEY :=
99 # RUNSAFELY - This program simply runs another program. If the program works
100 # correctly, this script has no effect, otherwise it will do things like print a
101 # stack trace of a core dump. It always returns "successful" so that tests will
102 # continue to be run.
103 ifdef PROGRAM_REQUIRED_TO_EXIT_OK
109 ifdef GET_STABLE_NUMBERS
110 RUNSAFELY := $(PROGDIR)/RunSafelyAndStable.sh $(RUNTIMELIMIT) $(EXIT_OK)
111 RUNSAFELYLOCAL := $(PROGDIR)/RunSafelyAndStable.sh $(RUNTIMELIMIT) $(EXIT_OK)
113 RUNSAFELY := $(PROGDIR)/RunSafely.sh
114 RUNSAFELYLOCAL := $(PROGDIR)/RunSafely.sh
117 RUNSAFELY := $(RUNSAFELY) -r $(REMOTE_HOST)
124 RUNSAFELY := $(RUNSAFELY) -l $(REMOTE_USER)
128 RUNSAFELY := $(RUNSAFELY) -rc $(REMOTE_CLIENT)
132 RUNSAFELY := $(RUNSAFELY) -rp "$(REMOTE_PORT)"
136 RUNSAFELY := $(RUNSAFELY) -u $(RUNUNDER)
140 RUNSAFELY := $(RUNSAFELY) -t "$(TIMEIT)"
141 RUNSAFELYLOCAL := $(RUNSAFELYLOCAL) -t "$(TIMEIT)"
144 RUNSAFELY := $(RUNSAFELY) $(RUNTIMELIMIT) $(EXIT_OK)
145 RUNSAFELYLOCAL := $(RUNSAFELYLOCAL) $(RUNTIMELIMIT) $(EXIT_OK)
148 RUNTOOLSAFELY := $(PROGDIR)/RunToolSafely.sh $(RUNTIMELIMIT)
150 ifndef STDIN_FILENAME
151 STDIN_FILENAME := /dev/null
155 # Targets to build for the default target...
158 # We will be working in the Output directory...
159 PREFIXED_PROGRAMS_TO_TEST := $(addprefix Output/,$(PROGRAMS_TO_TEST))
161 # Completed bytecode for a program
162 BYTECODE := $(addsuffix .llvm.bc, $(PREFIXED_PROGRAMS_TO_TEST))
164 LLCCODEGEN := $(addsuffix .llc.s, $(PREFIXED_PROGRAMS_TO_TEST))
165 LLCBETACODEGEN := $(addsuffix .llc-beta.s, $(PREFIXED_PROGRAMS_TO_TEST))
166 CBECODEGEN := $(addsuffix .cbe, $(PREFIXED_PROGRAMS_TO_TEST))
168 # Output produced by programs run
169 GCCOUTPUT := $(addsuffix .ll, $(addprefix Output/,$basename $(Source)))
170 NATOUTPUT := $(addsuffix .out-nat, $(PREFIXED_PROGRAMS_TO_TEST))
171 LLIOUTPUT := $(addsuffix .out-lli, $(PREFIXED_PROGRAMS_TO_TEST))
172 JITOUTPUT := $(addsuffix .out-jit, $(PREFIXED_PROGRAMS_TO_TEST))
173 JITBETAOUTPUT := $(addsuffix .out-jit-beta, $(PREFIXED_PROGRAMS_TO_TEST))
174 LLCOUTPUT := $(addsuffix .out-llc, $(PREFIXED_PROGRAMS_TO_TEST))
175 LLCBETAOUTPUT := $(addsuffix .out-llc-beta, $(PREFIXED_PROGRAMS_TO_TEST))
176 CBEOUTPUT := $(addsuffix .out-cbe, $(PREFIXED_PROGRAMS_TO_TEST))
178 # Diffs of program runs vs the native program
179 LLIDIFFS := $(addsuffix .diff-lli, $(PREFIXED_PROGRAMS_TO_TEST))
180 JITDIFFS := $(addsuffix .diff-jit, $(PREFIXED_PROGRAMS_TO_TEST))
181 JITBETADIFFS := $(addsuffix .diff-jit-beta, $(PREFIXED_PROGRAMS_TO_TEST))
182 LLCDIFFS := $(addsuffix .diff-llc, $(PREFIXED_PROGRAMS_TO_TEST))
183 LLCBETADIFFS := $(addsuffix .diff-llc-beta, $(PREFIXED_PROGRAMS_TO_TEST))
184 OPTBETADIFFS := $(addsuffix .diff-opt-beta, $(PREFIXED_PROGRAMS_TO_TEST))
185 CBEDIFFS := $(addsuffix .diff-cbe, $(PREFIXED_PROGRAMS_TO_TEST))
187 # Profiles for the program.
188 PROFOUTPUT := $(addsuffix .prof, $(PREFIXED_PROGRAMS_TO_TEST))
189 PRINTPROFOUTPUT := $(addsuffix .printprof, $(PREFIXED_PROGRAMS_TO_TEST))
191 # Build Program outputs:
192 .PRECIOUS: Output/%.out-simple
193 .PRECIOUS: Output/%.out-lli Output/%.out-jit Output/%.out-llc Output/%.out-llc-beta
194 .PRECIOUS: Output/%.out-nat Output/%.out-cbe Output/%.out-opt-beta
196 # Build diffs from the output...
197 .PRECIOUS: Output/%.diff-simple
198 .PRECIOUS: Output/%.diff-lli Output/%.diff-jit Output/%.diff-opt-beta
199 .PRECIOUS: Output/%.diff-llc Output/%.diff-llc-beta Output/%.diff-cbe
202 # Regardless of what other options are specified, build the program's bytecode
205 profile:: $(PROFOUTPUT)
206 print-profile:: $(PRINTPROFOUTPUT)
213 ifdef DISABLE_FOR_LLVM_PROGRAMS
218 DISABLE_LLC_DIFFS = 1
224 DISABLE_CBE_DIFFS = 1
228 ifndef DISABLE_LLC_DIFFS
231 all:: $(LLCBETADIFFS)
233 endif#DISABLE_LLC_DIFFS
235 ifndef DISABLE_CBE_DIFFS
243 all:: $(JITBETADIFFS)
253 ifeq ($(ARCH),PowerPC)
254 LLCBETAOPTION := -regalloc=local -O0
259 LLCBETAOPTION := -sched=list-td
260 # -enable-alpha-FTOI -enable-alpha-intfpdiv
263 LLCBETAOPTION := -sched=simple
265 ifeq ($(ARCH),x86_64)
266 LLCBETAOPTION := -combiner-alias-analysis
269 LLCBETAOPTION := -combiner-alias-analysis
272 LLCBETAOPTION := -enable-sparc-v9-insts
281 print-llcbeta-option:
282 @echo $(LLCBETAOPTION)
284 # Given an unoptimized bytecode file that is a simple linkage of all
285 # the program's bytecode files, optimize the program using the
286 # standard compilation optimizations.
287 $(PROGRAMS_TO_TEST:%=Output/%.linked.bc): \
288 Output/%.linked.bc: Output/%.linked.rbc $(LOPT)
289 $(VERB) $(RM) -f $(CURDIR)/$@.info
290 $(RUNTOOLSAFELY) $(LOPT) -std-compile-opts -info-output-file=$(CURDIR)/$@.info $(STATS) $(EXTRA_LOPT_OPTIONS) $< -o $@
292 $(PROGRAMS_TO_TEST:%=Output/%.llvm.stripped.bc): \
293 Output/%.llvm.stripped.bc: Output/%.llvm.bc $(LOPT)
294 $(RUNTOOLSAFELY) $(LOPT) -mstrip $< -o $@
296 $(PROGRAMS_TO_TEST:%=Output/%.linked.optbeta.bc): \
297 Output/%.linked.optbeta.bc: Output/%.linked.rbc $(LOPT)
298 $(VERB) $(RM) -f $(CURDIR)/$@.info
299 $(RUNTOOLSAFELY) $(LOPT) $(OPTBETAOPTIONS) -info-output-file=$(CURDIR)/$@.info $(STATS) $< -o $@
302 ifndef DISABLE_FOR_LLVM_PROGRAMS
303 # Rule to produce final program bytecode file from linked, optimized, bytecode.
304 # Link the program to the libraries it uses, then perform postlink
307 $(PROGRAMS_TO_TEST:%=Output/%.llvm.bc): \
308 Output/%.llvm.bc: Output/%.linked.bc $(LLVM_LDDPROG)
309 $(RUNTOOLSAFELY) $(LLVMLD) -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
310 $(EXTRA_LINKTIME_OPT_FLAGS) $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.llvm
311 ifneq ($(OPTPASSES),)
312 $(RUNTOOLSAFELY) $(LOPT) -q $(OPTPASSES) $@ -o $@.tmp
316 $(PROGRAMS_TO_TEST:%=Output/%.llvm): \
317 Output/%.llvm: Output/%.linked.bc $(LLVMLD)
318 $(RUNTOOLSAFELY) $(LLVMLD) -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
319 $(EXTRA_LINKTIME_OPT_FLAGS) $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.llvm
320 ifneq ($(OPTPASSES),)
321 $(RUNTOOLSAFELY) $(LOPT) -q $(OPTPASSES) $@ -o $@.tmp
325 $(PROGRAMS_TO_TEST:%=Output/%.llvm.optbeta.bc): \
326 Output/%.llvm.optbeta.bc: Output/%.linked.optbeta.bc $(LLVMLD)
327 $(RUNTOOLSAFELY) $(LLVMLD) -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
328 $(EXTRA_LINKTIME_OPT_FLAGS) $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.llvm.optbeta
330 $(PROGRAMS_TO_TEST:%=Output/%.llvm.optbeta): \
331 Output/%.llvm.optbeta: Output/%.linked.optbeta.bc $(LLVMLD)
332 $(RUNTOOLSAFELY) $(LLVMLD) -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
333 $(EXTRA_LINKTIME_OPT_FLAGS) $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.llvm.optbeta
335 $(PROGRAMS_TO_TEST:%=Output/%.noopt-llvm.bc): \
336 Output/%.noopt-llvm.bc: Output/%.linked.rbc $(LLVMLD)
337 $(RUNTOOLSAFELY) $(LLVMLD) -disable-opt -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
338 $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.noopt-llvm
340 $(PROGRAMS_TO_TEST:%=Output/%.noopt-llvm): \
341 Output/%.noopt-llvm: Output/%.linked.rbc $(LLVMLD)
342 $(RUNTOOLSAFELY) $(LLVMLD) -disable-opt -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
343 $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.noopt-llvm
346 $(PROGRAMS_TO_TEST:%=Output/%.nollvm-ldopt-llvm.bc): \
347 Output/%.nollvm-ldopt-llvm.bc: Output/%.linked.bc $(LLVMLD)
348 $(RUNTOOLSAFELY) $(LLVMLD) -disable-opt -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
349 $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.nollvm-ldopt-llvm
351 $(PROGRAMS_TO_TEST:%=Output/%.nollvm-ldopt-llvm): \
352 Output/%.nollvm-ldopt-llvm: Output/%.linked.rbc $(LLVMLD)
353 $(RUNTOOLSAFELY) $(LLVMLD) -disable-opt -info-output-file=$(CURDIR)/$@.info $(STATS) $< \
354 $(LLVMLD_FLAGS) -lc $(LIBS) -o Output/$*.nollvm-ldopt-llvm
356 endif # ifndef DISABLE_FOR_LLVM_PROGRAMS
358 # Disable asm-verbose. This can slow down compilation and is not what the
359 # compilers default to using.
360 LLCFLAGS += -asm-verbose=false
362 # If the program requires exception handling support, enable (potentially
363 # expensive) support for it.
364 ifdef REQUIRES_EH_SUPPORT
365 # PPC and X86 support DWARF exceptions, for everything else, default to SJLJ
366 # -enable-eh is no longer required to get DWARF exceptions.
367 ifneq ($(ARCH),PowerPC)
369 ifneq ($(ARCH),x86_64)
370 LLCFLAGS += -enable-correct-eh-support
371 LLVMLD_FLAGS += -disable-inlining
377 # llc optimization level
379 LLCFLAGS += $(LLC_OPTFLAGS)
382 # Pass target specific llc flags
383 ifdef TARGET_LLCFLAGS
384 LLCFLAGS += $(TARGET_LLCFLAGS)
387 LLCFLAGS += $(EXTRA_LLCFLAGS)
390 # It is important to link C++ programs with G++ so look for -lstdc++ in LDFLAGS
391 # and set the PROGRAMLD variable to the correct compiler interface to use.
392 # Note that LDFLAGS must already be defined at the time this file is included,
393 # or this will not work.
394 ifneq ($(filter -lstdc++,$(LDFLAGS)),)
401 # Rules to compile the program for the C Back End
403 $(PROGRAMS_TO_TEST:%=Output/%.cbe.c): \
404 Output/%.cbe.c: Output/%.llvm.bc $(LLC)
405 $(VERB) $(RM) -f $(CURDIR)/$@.info
406 $(RUNTOOLSAFELY) $(LLC) $(LLCFLAGS) -march=c $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
408 $(PROGRAMS_TO_TEST:%=Output/%.cbe): \
409 Output/%.cbe: Output/%.cbe.c
410 -$(RUNTOOLSAFELY) $(CC) $< -o $@ $(LDFLAGS) $(CFLAGS) $(OPTFLAGS) -fno-strict-aliasing -fno-inline $(X_TARGET_FLAGS) $(LIBS)
413 # Compile a linked program to machine code with LLC.
415 $(PROGRAMS_TO_TEST:%=Output/%.llc.s): \
416 Output/%.llc.s: Output/%.llvm.bc $(LLC)
417 $(VERB) $(RM) -f $(CURDIR)/$@.info
418 $(RUNTOOLSAFELY) $(LLC) $(LLCFLAGS) $(LLCOPTION) $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
420 $(PROGRAMS_TO_TEST:%=Output/%.llc-beta.s): \
421 Output/%.llc-beta.s: Output/%.llvm.bc $(LLC)
422 $(VERB) $(RM) -f $(CURDIR)/$@.info
423 $(RUNTOOLSAFELY) $(LLC) $(LLCFLAGS) $(LLCBETAOPTION) $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
425 $(PROGRAMS_TO_TEST:%=Output/%.opt-beta.s): \
426 Output/%.opt-beta.s: Output/%.llvm.optbeta.bc $(LLC)
427 $(VERB) $(RM) -f $(CURDIR)/$@.info
428 $(RUNTOOLSAFELY) $(LLC) $(LLCFLAGS) $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
430 # On darwin, pass -force_cpusubtype_ALL to allow all ppc instructions.
431 ifeq ($(ARCH),PowerPC)
432 LLCASSEMBLERFLAGS = -force_cpusubtype_ALL
434 # On sparc, pass -mcpu=v9 to allow all V9 instructions, even in 32-bit mode.
436 LLCASSEMBLERFLAGS = -mcpu=v9
439 # Generate a .o file from the llvm.bc file with the integrated assembler.
440 $(PROGRAMS_TO_TEST:%=Output/%.llc.o): \
441 Output/%.llc.o: Output/%.llvm.bc $(LLC)
442 $(VERB) $(RM) -f $(CURDIR)/$@.info
443 $(RUNTOOLSAFELY) $(LLC) $(LLCFLAGS) -filetype=obj $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
445 $(PROGRAMS_TO_TEST:%=Output/%.llc-beta.o): \
446 Output/%.llc-beta.o: Output/%.llvm.bc $(LLC)
447 $(VERB) $(RM) -f $(CURDIR)/$@.info
448 $(RUNTOOLSAFELY) $(LLC) $(LLCFLAGS) $(LLCBETAOPTION) -filetype=obj $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
450 $(PROGRAMS_TO_TEST:%=Output/%.opt-beta.o): \
451 Output/%.opt-beta.o: Output/%.llvm.optbeta.bc $(LLC)
452 $(VERB) $(RM) -f $(CURDIR)/$@.info
453 $(RUNTOOLSAFELY) $(LLC) $(LLCFLAGS) -filetype=obj $< -o $@ -info-output-file=$(CURDIR)/$@.info $(STATS)
456 ifdef TEST_INTEGRATED_ASSEMBLER
458 # Link an LLVM-linked program using the system linker.
459 $(PROGRAMS_TO_TEST:%=Output/%.llc): \
460 Output/%.llc: Output/%.llc.o
461 -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) $(LDFLAGS)
463 $(PROGRAMS_TO_TEST:%=Output/%.llc-beta): \
464 Output/%.llc-beta: Output/%.llc-beta.o
465 -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) $(LDFLAGS)
467 $(PROGRAMS_TO_TEST:%=Output/%.opt-beta): \
468 Output/%.opt-beta: Output/%.opt-beta.o
469 -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) $(LDFLAGS)
473 # Assemble/Link an LLVM-linked program using the system assembler and linker.
475 $(PROGRAMS_TO_TEST:%=Output/%.llc): \
476 Output/%.llc: Output/%.llc.s
477 -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) $(LDFLAGS)
479 $(PROGRAMS_TO_TEST:%=Output/%.llc-beta): \
480 Output/%.llc-beta: Output/%.llc-beta.s
481 -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) $(LDFLAGS)
483 $(PROGRAMS_TO_TEST:%=Output/%.opt-beta): \
484 Output/%.opt-beta: Output/%.opt-beta.s
485 -$(PROGRAMLD) $< -o $@ $(LLCLIBS) $(LLCASSEMBLERFLAGS) $(X_TARGET_FLAGS) $(LDFLAGS)
490 # Rules to execute the program
493 # Note, these must be lazily expanded by make (for EXTRA_LLIFLAGS to work, it is
494 # redefined in other Makefiles).
495 LLI_OPTS = -force-interpreter=true --disable-core-files
496 JIT_OPTS = -force-interpreter=false --disable-core-files
498 # lli optimization level
500 LLI_OPTFLAGS += $(LLI_OPTFLAGS)
501 JIT_OPTFLAGS += $(LLI_OPTFLAGS)
504 # Pass target specific lli flags
505 ifdef TARGET_LLIFLAGS
506 LLI_OPTS += $(TARGET_LLIFLAGS)
507 JIT_OPTS += $(TARGET_LLIFLAGS)
510 # EXTRA_LLIFLAGS is used by the nighly tester to add arugments to invocations of
511 # the JIT and LLI in order to get timing info and statistics.
512 ifndef EXTRA_LLIFLAGS
515 LLI_OPTS += $(EXTRA_LLIFLAGS)
516 JIT_OPTS += $(EXTRA_LLIFLAGS)
518 # If the program requires exception handling support, enable (potentially
519 # expensive) support for it.
520 ifdef REQUIRES_EH_SUPPORT
521 JIT_OPTS += -enable-correct-eh-support
524 # If the program wants its output hashed, add an output filter.
525 ifdef ENABLE_HASHED_PROGRAM_OUTPUT
526 ifdef HASH_PROGRAM_OUTPUT
527 PROGRAM_OUTPUT_FILTER := $(PROGDIR)/HashProgramOutput.sh
531 native:: $(PROGRAMS_TO_TEST:%=Output/%.native)
532 stripped-bytecode:: $(PROGRAMS_TO_TEST:%=Output/%.llvm.stripped.bc)
534 ifndef PROGRAMS_HAVE_CUSTOM_RUN_RULES
536 # Rules to build the test output...
537 ifndef USE_PRECOMPILED_BYTECODE
538 ifndef USE_REFERENCE_OUTPUT
540 $(PROGRAMS_TO_TEST:%=Output/%.out-nat): \
541 Output/%.out-nat: Output/%.native
542 $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
543 ifdef PROGRAM_OUTPUT_FILTER
544 $(PROGRAM_OUTPUT_FILTER) $@
546 ifdef UPDATE_REFERENCE_OUTPUT
547 ifeq ($(REFERENCE_OUTPUT_KEY),)
548 cp $@ $(PROJ_SRC_DIR)/$*.reference_output
550 if [ ! -f $(PROJ_SRC_DIR)/$*.reference_output ]; then \
551 echo "error: no normal reference output!"; \
553 elif (! diff -q $@ $(PROJ_SRC_DIR)/$*.reference_output); then \
554 cp $@ $(PROJ_SRC_DIR)/$*.reference_output.$(REFERENCE_OUTPUT_KEY); \
556 echo "no need to update $(REFERENCE_OUTPUT_KEY) reference," \
557 "matches normal reference!"; \
565 $(PROGRAMS_TO_TEST:%=Output/%.out-simple): \
566 Output/%.out-simple: Output/%.simple
567 $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
568 ifdef PROGRAM_OUTPUT_FILTER
569 $(PROGRAM_OUTPUT_FILTER) $@
572 $(PROGRAMS_TO_TEST:%=Output/%.out-lli): \
573 Output/%.out-lli: Output/%.llvm.bc $(LLI)
574 $(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) -info-output-file=$(CURDIR)/$@.info $(STATS) $(LLI_OPTS) $< $(RUN_OPTIONS)
575 ifdef PROGRAM_OUTPUT_FILTER
576 $(PROGRAM_OUTPUT_FILTER) $@
579 $(PROGRAMS_TO_TEST:%=Output/%.out-jit): \
580 Output/%.out-jit: Output/%.llvm.bc $(LLI)
581 $(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) -info-output-file=$(CURDIR)/$@.info $(STATS) $(JIT_OPTS) $< $(RUN_OPTIONS)
582 ifdef PROGRAM_OUTPUT_FILTER
583 $(PROGRAM_OUTPUT_FILTER) $@
586 $(PROGRAMS_TO_TEST:%=Output/%.out-jit-beta): \
587 Output/%.out-jit-beta: Output/%.llvm.bc $(LLI)
588 $(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) -info-output-file=$(CURDIR)/$@.info $(STATS) $(LLCBETAOPTION) $(JIT_OPTS) $< $(RUN_OPTIONS)
589 ifdef PROGRAM_OUTPUT_FILTER
590 $(PROGRAM_OUTPUT_FILTER) $@
593 $(PROGRAMS_TO_TEST:%=Output/%.out-llc): \
594 Output/%.out-llc: Output/%.llc
595 $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
596 ifdef PROGRAM_OUTPUT_FILTER
597 $(PROGRAM_OUTPUT_FILTER) $@
600 $(PROGRAMS_TO_TEST:%=Output/%.out-llc-beta): \
601 Output/%.out-llc-beta: Output/%.llc-beta
602 $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
603 ifdef PROGRAM_OUTPUT_FILTER
604 $(PROGRAM_OUTPUT_FILTER) $@
607 $(PROGRAMS_TO_TEST:%=Output/%.out-opt-beta): \
608 Output/%.out-opt-beta: Output/%.opt-beta
609 $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
610 ifdef PROGRAM_OUTPUT_FILTER
611 $(PROGRAM_OUTPUT_FILTER) $@
614 $(PROGRAMS_TO_TEST:%=Output/%.out-cbe): \
615 Output/%.out-cbe: Output/%.cbe
616 $(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
617 ifdef PROGRAM_OUTPUT_FILTER
618 $(PROGRAM_OUTPUT_FILTER) $@
621 # The RunSafely.sh script puts an "exit <retval>" line at the end of
622 # the program's output. We have to make bugpoint do the same thing
623 # or else it will get false positives when it diff's the reference
624 # output with the program's output.
625 BUGPOINT_OPTIONS += -append-exit-code
627 # Support remote execution
629 BUGPOINT_OPTIONS += -remote-host=$(REMOTE_HOST)
631 BUGPOINT_OPTIONS += -remote-port=$(REMOTE_PORT)
634 BUGPOINT_OPTIONS += -remote-user=$(REMOTE_USER)
637 BUGPOINT_OPTIONS += -remote-client=$(REMOTE_CLIENT)
641 # If a tolerance is set, pass it off to bugpoint
643 BUGPOINT_OPTIONS += -rel-tolerance $(FP_TOLERANCE)
645 ifdef FP_ABSTOLERANCE
646 BUGPOINT_OPTIONS += -abs-tolerance $(FP_ABSTOLERANCE)
649 ifdef DISABLE_LOOP_EXTRACT
650 BUGPOINT_OPTIONS += -disable-loop-extraction
653 ifdef DISABLE_BLOCK_EXTRACT
654 BUGPOINT_OPTIONS += -disable-block-extraction
657 # Give bugpoint information about LDFLAGS to pass down to the actual link stage
659 BUGPOINT_OPTIONS += $(LDFLAGS:%=-Xlinker=%) $(EXTRA_OPTIONS:%=-Xlinker=%)
661 # Specify stdin, reference output, and command line options for the program...
662 BUGPOINT_OPTIONS += -input=$(STDIN_FILENAME) -output=Output/$*.out-nat
663 BUGPOINT_OPTIONS += -timeout=$(RUNTIMELIMIT)
664 BUGPOINT_OPTIONS += --tool-args $(LLCFLAGS)
666 BUGPOINT_TOOL_OPTIONS = -gcc-tool-args -m32
668 ifeq ($(ARCH),x86_64)
669 BUGPOINT_TOOL_OPTIONS += -gcc-tool-args -m64
671 BUGPOINT_ARGS += --args -- $(RUN_OPTIONS)
673 # Rules to bugpoint the opt, llvm-ld, llc, or lli commands...
674 $(PROGRAMS_TO_TEST:%=Output/%.bugpoint-opt): \
675 Output/%.bugpoint-opt: Output/%.noopt-llvm.bc $(LBUGPOINT) \
677 PWD=$(CURDIR) $(LBUGPOINT) -llc-safe $< -std-compile-opts \
678 $(BUGPOINT_OPTIONS) $(BUGPOINT_TOOL_OPTIONS) $(BUGPOINT_ARGS)
680 $(PROGRAMS_TO_TEST:%=Output/%.bugpoint-llvm-ld): \
681 Output/%.bugpoint-llvm-ld: Output/%.nollvm-ldopt-llvm.bc $(LBUGPOINT) \
683 PWD=$(CURDIR) $(LBUGPOINT) -llc-safe $< -std-link-opts $(OPTPASSES) \
684 $(BUGPOINT_OPTIONS) $(BUGPOINT_TOOL_OPTIONS) $(BUGPOINT_ARGS)
686 $(PROGRAMS_TO_TEST:%=Output/%.bugpoint-llc): \
687 Output/%.bugpoint-llc: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
688 PWD=$(CURDIR) $(LBUGPOINT) $< -run-llc \
689 $(BUGPOINT_OPTIONS) $(BUGPOINT_TOOL_OPTIONS) $(BUGPOINT_ARGS)
691 $(PROGRAMS_TO_TEST:%=Output/%.bugpoint-llc-beta): \
692 Output/%.bugpoint-llc-beta: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
693 PWD=$(CURDIR) $(LBUGPOINT) $< -llc-safe \
694 $(BUGPOINT_OPTIONS) $(LLCBETAOPTION) $(BUGPOINT_TOOL_OPTIONS) $(BUGPOINT_ARGS)
696 $(PROGRAMS_TO_TEST:%=Output/%.bugpoint-opt-beta): \
697 Output/%.bugpoint-opt-beta: Output/%.linked.rbc $(LBUGPOINT) Output/%.out-nat
698 PWD=$(CURDIR) $(LBUGPOINT) -llc-safe $< \
699 $(BUGPOINT_OPTIONS) $(BUGPOINT_TOOL_OPTIONS) $(OPTBETAOPTIONS) $(BUGPOINT_ARGS)
701 $(PROGRAMS_TO_TEST:%=Output/%.bugpoint-jit): \
702 Output/%.bugpoint-jit: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
703 PWD=$(CURDIR) $(LBUGPOINT) $< -run-jit -safe-run-llc \
704 $(BUGPOINT_OPTIONS) $(BUGPOINT_TOOL_OPTIONS) $(BUGPOINT_ARGS)
706 $(PROGRAMS_TO_TEST:%=Output/%.bugpoint-cbe): \
707 Output/%.bugpoint-cbe: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
708 PWD=$(CURDIR) $(LBUGPOINT) $< -cbe-bug \
709 $(BUGPOINT_OPTIONS) $(BUGPOINT_TOOL_OPTIONS) $(BUGPOINT_ARGS)
711 $(PROGRAMS_TO_TEST:%=Output/%.misopt.out): \
712 Output/%.misopt.out: Output/%.linked.rbc $(LFINDMISOPT)
713 $(LFINDMISOPT) $< Output/$*.misopt "$(RUN_OPTIONS)" \
714 "$(STDIN_FILENAME)" > Output/$*.misopt.out 2>&1
719 rm -rf Output/misopt-*
721 LIBPROFILESO = $(LLVM_OBJ_ROOT)/Debug/lib/libprofile_rt.so
723 $(PROGRAMS_TO_TEST:%=Output/%.prof): \
724 Output/%.prof: Output/%.llvm-prof.bc Output/%.out-nat $(LIBPROFILESO)
726 $(RUNSAFELY) $(STDIN_FILENAME) Output/$*.out-prof $(LLI) $(JIT_OPTS) \
727 -fake-argv0 'Output/$*.llvm.bc' -load $(LIBPROFILESO) $< -llvmprof-output $@ $(RUN_OPTIONS)
728 @cmp -s Output/$*.out-prof Output/$*.out-nat || \
729 printf "***\n***\n*** WARNING: Output of profiled program (Output/$*.out-prof)\n*** doesn't match the output of the native program (Output/$*.out-nat)!\n***\n***\n";
735 # Rules to generate profiling information
737 $(PROGRAMS_TO_TEST:%=Output/%.llvm-prof.bc): \
738 Output/%.llvm-prof.bc: Output/%.llvm.bc
739 $(RUNTOOLSAFELY) $(LOPT) -insert-edge-profiling $< -o $@
741 $(PROGRAMS_TO_TEST:%=Output/%.printprof): \
742 Output/%.printprof: Output/%.llvm.bc Output/%.prof $(LPROF)
743 $(RUNTOOLSAFELY) $(LPROF) $< Output/$*.prof
747 # Rules to diff test output...
749 ifdef PROGRAM_IS_NONDETERMINISTIC
750 # If the program is non-deterministic, just touch $@
751 DIFFPROG = touch $@ \#
754 $(PROGRAMS_TO_TEST:%=Output/%.diff-simple): \
755 Output/%.diff-simple: Output/%.out-nat Output/%.out-simple
756 -$(DIFFPROG) simple $* $(HIDEDIFF)
758 $(PROGRAMS_TO_TEST:%=Output/%.diff-lli): \
759 Output/%.diff-lli: Output/%.out-nat Output/%.out-lli
760 -$(DIFFPROG) lli $* $(HIDEDIFF)
762 $(PROGRAMS_TO_TEST:%=Output/%.diff-jit): \
763 Output/%.diff-jit: Output/%.out-nat Output/%.out-jit
764 -$(DIFFPROG) jit $* $(HIDEDIFF)
766 $(PROGRAMS_TO_TEST:%=Output/%.diff-jit-beta): \
767 Output/%.diff-jit-beta: Output/%.out-nat Output/%.out-jit-beta
768 -$(DIFFPROG) jit-beta $* $(HIDEDIFF)
770 $(PROGRAMS_TO_TEST:%=Output/%.diff-llc): \
771 Output/%.diff-llc: Output/%.out-nat Output/%.out-llc
772 -$(DIFFPROG) llc $* $(HIDEDIFF)
774 $(PROGRAMS_TO_TEST:%=Output/%.diff-llc-beta): \
775 Output/%.diff-llc-beta: Output/%.out-nat Output/%.out-llc-beta
776 -$(DIFFPROG) llc-beta $* $(HIDEDIFF)
778 $(PROGRAMS_TO_TEST:%=Output/%.diff-opt-beta): \
779 Output/%.diff-opt-beta: Output/%.out-nat Output/%.out-opt-beta
780 -$(DIFFPROG) opt-beta $* $(HIDEDIFF)
782 $(PROGRAMS_TO_TEST:%=Output/%.diff-cbe): \
783 Output/%.diff-cbe: Output/%.out-nat Output/%.out-cbe
784 -$(DIFFPROG) cbe $* $(HIDEDIFF)
787 $(PROGRAMS_TO_TEST:%=Output/%.exe-simple): \
788 Output/%.exe-simple: Output/%.diff-simple
792 $(PROGRAMS_TO_TEST:%=Output/%.exe-lli): \
793 Output/%.exe-lli: Output/%.diff-lli
797 $(PROGRAMS_TO_TEST:%=Output/%.exe-jit): \
798 Output/%.exe-jit: Output/%.diff-jit
802 $(PROGRAMS_TO_TEST:%=Output/%.exe-jit-beta): \
803 Output/%.exe-jit-beta: Output/%.diff-jit-beta
807 $(PROGRAMS_TO_TEST:%=Output/%.exe-llc): \
808 Output/%.exe-llc: Output/%.diff-llc
812 $(PROGRAMS_TO_TEST:%=Output/%.exe-llc-beta): \
813 Output/%.exe-llc-beta: Output/%.diff-llc-beta
817 $(PROGRAMS_TO_TEST:%=Output/%.exe-opt-beta): \
818 Output/%.exe-opt-beta: Output/%.diff-opt-beta
822 $(PROGRAMS_TO_TEST:%=Output/%.exe-cbe): \
823 Output/%.exe-cbe: Output/%.diff-cbe
827 # Pseudo target to build just the bytecode file.
828 bytecode:: $(PROGRAMS_TO_TEST:%=Output/%.llvm.bc)
832 # Rules to support the USE_PRECOMPILED_BYTECODE setting If set, submakefiles
833 # will not know how to make output bytecode files for the programs in this
834 # directory. Instead, this makefile just copies them out of the bytecode
836 ifdef USE_PRECOMPILED_BYTECODE
838 # Calculate the directory we should copy the bytecode file into. This is
839 # relative to BYTECODE_REPOSITORY and the current directory this program is in.
841 CURDIR := $(shell cd .; pwd)
842 PROGDIR := $(shell cd $(LEVEL); pwd)
843 SRCDIR := $(BYTECODE_REPOSITORY)/$(subst $(PROGDIR),,$(CURDIR))
845 # Because we don't have source code, we cannot build a native version of the
846 # executable. Copy over pregenerated reference output.
847 $(PROGRAMS_TO_TEST:%=Output/%.out-nat): \
848 Output/%.out-nat: $(SRCDIR)/%.reference_output Output/.dir
851 $(PROGRAMS_TO_TEST:%=Output/%.LOC.txt): \
852 Output/%.LOC.txt: $(SRCDIR)/%.LOC.txt Output/.dir
856 # Rules to support USE_REFERENCE_OUTPUT.
857 ifdef USE_REFERENCE_OUTPUT
859 # If a reference output file is specified, use that.
860 ifdef REFERENCE_OUTPUT_FILE
862 Output/%.out-nat: $(REFERENCE_OUTPUT_FILE) Output/.dir
867 # Otherwise, pick the best reference output based on
868 # 'progamname.reference_output'.
870 # Note that this rule needs to be in both Makefile.programs and Makefile.spec.
871 Output/%.out-nat: Output/.dir
872 -if [ -f "$(PROJ_SRC_DIR)/$*.reference_output.$(REFERENCE_OUTPUT_KEY)" ]; then \
873 cp $(PROJ_SRC_DIR)/$*.reference_output.$(REFERENCE_OUTPUT_KEY) $@; \
874 elif [ -f "$(PROJ_SRC_DIR)/$*.reference_output" ]; then \
875 cp $(PROJ_SRC_DIR)/$*.reference_output $@; \
877 printf "WARNING: %s: %s\n" "NO REFERENCE OUTPUT (using default)" "$(PROJ_SRC_DIR)/$*.reference_output" > $@; \
878 cp $(PROJ_SRC_ROOT)/default.reference_output $@; \
887 # Support for the TEST= option... when TEST= is specified on the command line,
888 # the default target is the test target. Here we dispatch to a specific set of
891 test:: $(PROGRAMS_TO_TEST:%=test.$(TEST).%)
893 # AVAILABLE_TESTS - Compute the set of tests available for user help
895 TEST_FILES = $(wildcard $(PROGDIR)/TEST.*.Makefile) \
896 $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.*.Makefile)
897 AVAILABLE_TESTS = $(patsubst TEST.%.Makefile,%,$(notdir $(TEST_FILES)))
899 # If they just say 'make test' then we print out an error telling the user to
900 # specify a TEST= option.
901 $(PROGRAMS_TO_TEST:%=test..%): \
904 @echo "***************************************************************"
905 @echo " ERROR: you cannot type '$(MAKE) test' directly."
906 @echo " Instead, use '$(MAKE) TEST=X' where X is the name of a test."
907 @echo " Tests available: $(AVAILABLE_TESTS)"
908 @echo " Alternatively, just use '$(MAKE)' to run comparisons."
909 @echo "***************************************************************"
913 # Include all makefiles which define tests... These makefiles must define
914 # test.<testname>.% given input from Output/%.llvm.bc
917 TestMakefile := $(wildcard $(PROGDIR)/TEST.$(TEST).Makefile) \
918 $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.$(TEST).Makefile)
919 TestReport := $(wildcard $(PROGDIR)/TEST.$(TEST).report) \
920 $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.$(TEST).report)
921 TestGnuPlot := $(wildcard $(PROGDIR)/TEST.$(TEST).gnuplot) \
922 $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.$(TEST).gnuplot)
923 ifneq ($(strip $(TestMakefile)),)
924 -include $(TestMakefile)
926 $(PROGRAMS_TO_TEST:%=test.$(TEST).%): \
929 @echo "***************************************************************"
930 @echo " ERROR: Test '$(TEST)' is not a known test!"
931 @echo " Tests Available: $(AVAILABLE_TESTS)"
932 @echo " Test Program: $*"
933 @echo " Test Makefile: $(TestMakefile)"
934 @echo " Test Report: $(TestReport)"
935 @echo "***************************************************************"
942 # Rules for building a report from 'make report TEST=<x>'
944 GENERATEREPORT := $(PROJ_SRC_ROOT)/GenerateReport.pl
946 ifndef ENABLE_PARALLEL_REPORT
947 FORCE_SERIAL_ARG := -j1
950 report.$(TEST).raw.out: $(REPORT_DEPENDENCIES) $(TestMakefile)
951 $(MAKE) $(FORCE_SERIAL_ARG) TEST=$(TEST)
952 find . -name \*.$(TEST).report.txt -exec cat {} \; | tee $@
954 ifneq ($(TestReport),)
955 report.$(TEST).txt: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
956 $(GENERATEREPORT) $(TestReport) < $< > $@
958 report.$(TEST).html: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
959 $(GENERATEREPORT) -html $(TestReport) < $< > $@
961 report.$(TEST).tex: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
962 $(GENERATEREPORT) -latex $(TestReport) < $< > $@
964 report.$(TEST).csv: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
965 $(GENERATEREPORT) -csv $(TestReport) < $< > $@
967 report.graphs: report.$(TEST).raw.out $(TestReport) $(TestGnuPlot) $(GENERATEREPORT)
968 $(GENERATEREPORT) -graphs $(TestReport) < $<
969 gnuplot $(TestGnuPlot)
971 report: report.$(TEST).txt
974 report.html: report.$(TEST).html
976 report.csv: report.$(TEST).csv
978 report.tex: report.$(TEST).tex
981 report.$(TEST).txtonly: report.$(TEST).raw.out $(TestReport)
982 @cat report.$(TEST).raw.out
984 report.dbgopt.header:
985 @echo "***************************************************************"
986 @echo " Test failures in this report indicates that code generation"
987 @echo " for the test case is influenced by presense of debugging"
988 @echo " information."
990 @echo " Run following commands to investigate failures."
991 @echo " prompt> clang -g -fdebug-disable-debug-info-print -Os -S foo.c -o foo.first.s"
992 @echo " prompt> clang -Os -S foo.c -o foo.second.s"
993 @echo " prompt> diff foo.first.s foo.second.s"
994 @echo "***************************************************************"
996 report.dbgopt: report.dbgopt.header report.$(TEST).txtonly
1001 rm -f report.*.raw.out report.*.txt report.*.html report.*.tex report.*.csv