##===- llvm-test/Makefile.programs -------------------------*- Makefile -*-===##
#
# This makefile contains all of the makefile machinery that is common to
# building stuff in this directory.  This script can be used in two
# different ways.  The main job of this is to take executables for the following
# targets:
#
#   1. The native platform compiler
#   2. LLVM Bytecode Compiler + LLI interpreter (if ENABLE_LLI is enabled)
#   3. LLVM Bytecode Compiler + LLC static compiler
#   4. LLVM Bytecode Compiler + C Backend + Native Sun Compiler
#   5. LLVM Bytecode Compiler + LLI Just-In-Time Compiler
#
# Running them, and then diffing the output.  If there are any failures, they
# are flagged.  The other mode is used in conjunction with the TEST=<testname>
# argument on the make command line.  In this case, a Makefile.TEST.<testname>
# makefile is used to run tests on the program (see below).
#
# Input to this makefile should be the PROGRAMS_TO_TEST variable, which contains
# a list of programs that should be run.  The makefile can also optionally
# specify a STDIN_FILENAME variable, which contains a filename that is piped
# into the program as it is being executed.
#
#  FIXME: There should be a way to specify the command line for a program
#
##===----------------------------------------------------------------------===##
#
#                            Running Custom Tests
#
# This makefile provides facilities for defining custom tests that are to be run
# on all of the programs in the test suite.  In order to define one of these
# tests, create a llvm-test/Makefile.TEST.<testname> file.  This file
# should define a rule 'test.<testname>.%' which is run once for each program in
# the suite (the % passed in is the program name).  For a simple example, see
# Makefile.TEST.example.
#
##===----------------------------------------------------------------------===##

# Dependencies on header files need to be determined explicitly because
# we do not automatically compute dependencies
INCLUDES := $(ExtraHeaders) $(wildcard $(SourceDir)/*.h)

include $(LEVEL)/Makefile.tests

.PRECIOUS: Output/%.llvm Output/%.native Output/%.llc Output/%.llc.s
.PRECIOUS: Output/%.cbe Output/%.cbe.c Output/%.llvm.bc
.PRECIOUS: Output/%.linked.bc 

PROGDIR = $(BUILD_SRC_ROOT)

#
# Scripts in the this directory...
#

# TIMEPROG - The program used to get timing results for a program
TIMEPROG := $(PROGDIR)/TimeProgram.sh

TOLERANCEOPT :=
ifdef FP_TOLERANCE
TOLERANCEOPT += -r $(FP_TOLERANCE)
endif
ifdef FP_ABSTOLERANCE
TOLERANCEOPT += -a $(FP_ABSTOLERANCE)
endif

# DIFFPROG - The program used to diff the output
DIFFPROG := $(PROGDIR)/DiffOutput.sh "$(LLVMTOOLCURRENT)/fpcmp $(TOLERANCEOPT)"

# RUNTIMELIMIT - The number of seconds we should wait before certain events
# timeout.  This is overridable on the commandline or in tests makefiles.
#
ifndef RUNTIMELIMIT
  RUNTIMELIMIT := 500
endif

# RUNSAFELY - This program simply runs another program.  If the program works
# correctly, this script has no effect, otherwise it will do things like print a
# stack trace of a core dump.  It always returns "successful" so that tests will
# continue to be run.
ifndef GET_STABLE_NUMBERS
RUNSAFELY := $(PROGDIR)/RunSafely.sh $(RUNTIMELIMIT)
else
RUNSAFELY := $(PROGDIR)/RunSafelyAndStable.sh $(RUNTIMELIMIT)
endif

ifndef STDIN_FILENAME
STDIN_FILENAME := /dev/null
endif

#
# Targets to build for the default target...
#

# We will be working in the Output directory... 
PREFIXED_PROGRAMS_TO_TEST := $(addprefix Output/,$(PROGRAMS_TO_TEST))

# Completed bytecode for a program
BYTECODE   := $(addsuffix .llvm.bc, $(PREFIXED_PROGRAMS_TO_TEST))

LLCCODEGEN := $(addsuffix .llc.s,   $(PREFIXED_PROGRAMS_TO_TEST))
LLCLSCODEGEN := $(addsuffix .llc-ls.s,   $(PREFIXED_PROGRAMS_TO_TEST))
CBECODEGEN := $(addsuffix .cbe,     $(PREFIXED_PROGRAMS_TO_TEST))

# Output produced by programs run
GCCOUTPUT := $(addsuffix .ll,       $(addprefix Output/,$basename $(Source)))
NATOUTPUT := $(addsuffix .out-nat,  $(PREFIXED_PROGRAMS_TO_TEST))
LLIOUTPUT := $(addsuffix .out-lli,  $(PREFIXED_PROGRAMS_TO_TEST))
JITOUTPUT := $(addsuffix .out-jit,  $(PREFIXED_PROGRAMS_TO_TEST))
JITLSOUTPUT := $(addsuffix .out-jit-ls,  $(PREFIXED_PROGRAMS_TO_TEST))
LLCOUTPUT := $(addsuffix .out-llc,  $(PREFIXED_PROGRAMS_TO_TEST))
LLCLSOUTPUT := $(addsuffix .out-llc-ls,  $(PREFIXED_PROGRAMS_TO_TEST))
CBEOUTPUT := $(addsuffix .out-cbe,  $(PREFIXED_PROGRAMS_TO_TEST))

# Diffs of program runs vs the native program
LLIDIFFS  := $(addsuffix .diff-lli, $(PREFIXED_PROGRAMS_TO_TEST))
JITDIFFS  := $(addsuffix .diff-jit, $(PREFIXED_PROGRAMS_TO_TEST))
JITLSDIFFS  := $(addsuffix .diff-jit-ls, $(PREFIXED_PROGRAMS_TO_TEST))
LLCDIFFS  := $(addsuffix .diff-llc, $(PREFIXED_PROGRAMS_TO_TEST))
LLCLSDIFFS  := $(addsuffix .diff-llc-ls, $(PREFIXED_PROGRAMS_TO_TEST))
CBEDIFFS  := $(addsuffix .diff-cbe, $(PREFIXED_PROGRAMS_TO_TEST))

# Profiles for the program.
PROFOUTPUT := $(addsuffix .prof, $(PREFIXED_PROGRAMS_TO_TEST))
PRINTPROFOUTPUT := $(addsuffix .printprof, $(PREFIXED_PROGRAMS_TO_TEST))

# Build Program outputs:
.PRECIOUS: Output/%.out-lli Output/%.out-jit Output/%.out-llc Output/%.out-llc-ls
.PRECIOUS: Output/%.out-nat Output/%.out-cbe

# Build diffs from the output...
.PRECIOUS: Output/%.diff-lli Output/%.diff-jit
.PRECIOUS: Output/%.diff-llc Output/%.diff-llc-ls Output/%.diff-cbe


# Regardless of what other options are specified, build the program's bytecode
# representation.
all:: $(BYTECODE)
profile:: $(PROFOUTPUT)
print-profile:: $(PRINTPROFOUTPUT)


ifdef RUN_GCC_ONLY
DISABLE_DIFFS = 1
all:: $(GCCOUTPUT)
endif

ifdef DISABLE_FOR_LLVM_PROGRAMS
  DISABLE_DIFFS := 1
endif

ifndef DISABLE_LLC
all:: $(LLCCODEGEN)
else
DISABLE_LLC_DIFFS = 1
endif

ifndef DISABLE_CBE
all:: $(CBECODEGEN)
else
DISABLE_CBE_DIFFS = 1
endif

ifndef DISABLE_DIFFS
ifndef DISABLE_LLC_DIFFS
all:: $(LLCDIFFS)
ifdef ENABLE_LINEARSCAN
all:: $(LLCLSDIFFS)
endif
endif#DISABLE_LLC_DIFFS

ifndef DISABLE_CBE_DIFFS
all:: $(CBEDIFFS)
endif

ifdef TARGET_HAS_JIT
ifndef DISABLE_JIT
all:: $(JITDIFFS)
ifdef ENABLE_LINEARSCAN
all:: $(JITLSDIFFS)
endif
endif#DISABLE_JIT
endif#TARGET_HAS_JIT

ifdef ENABLE_LLI
all:: $(LLIDIFFS)
endif
endif#DISABLE_DIFFS

# Given a version of the entire program linked together into a single unit of
# raw output from the C frontend, optimize it.
$(PROGRAMS_TO_TEST:%=Output/%.linked.bc): \
Output/%.linked.bc: Output/%.linked.rll $(LGCCAS)
	-$(LGCCAS) $(STATS) $< -o $@

$(PROGRAMS_TO_TEST:%=Output/%.llvm.stripped.bc): \
Output/%.llvm.stripped.bc: Output/%.llvm.bc $(LOPT)
	-$(LOPT) -mstrip $< -o $@ -f


ifndef DISABLE_FOR_LLVM_PROGRAMS
# Rule to produce final program bytecode file from linked, optimized, bytecode.
# Link the program to the libraries it uses, then perform postlink
# optimization...

$(PROGRAMS_TO_TEST:%=Output/%.llvm.bc): \
Output/%.llvm.bc: Output/%.linked.bc $(LGCCLDPROG)
	-$(LGCCLD) $(STATS) $< -lc $(LIBS) -lcrtend -o Output/$*.llvm
ifneq ($(OPTPASSES),)
	-$(LOPT) -q $(OPTPASSES) < $@ > $@.tmp
	$(MV) -f $@.tmp $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.llvm): \
Output/%.llvm: Output/%.linked.bc $(LGCCLDPROG)
	-$(LGCCLD) $(STATS) $< -lc $(LIBS) -lcrtend -o Output/$*.llvm
ifneq ($(OPTPASSES),)
	-$(LOPT) -q $(OPTPASSES) < $@ > $@.tmp
	$(MV) -f $@.tmp $@
endif

$(PROGRAMS_TO_TEST:%=Output/%.noopt-llvm.bc): \
Output/%.noopt-llvm.bc: Output/%.linked.rbc $(LGCCLDPROG)
	-$(LGCCLD) -disable-opt $(STATS) $< -lc $(LIBS) -lcrtend -o Output/$*.noopt-llvm

$(PROGRAMS_TO_TEST:%=Output/%.noopt-llvm): \
Output/%.noopt-llvm: Output/%.linked.rbc $(LGCCLDPROG)
	-$(LGCCLD) -disable-opt $(STATS) $< -lc $(LIBS) -lcrtend -o Output/$*.noopt-llvm


$(PROGRAMS_TO_TEST:%=Output/%.nogccldopt-llvm.bc): \
Output/%.nogccldopt-llvm.bc: Output/%.linked.bc $(LGCCLDPROG)
	-$(LGCCLD) -disable-opt $(STATS) $< -lc $(LIBS) -lcrtend -o Output/$*.nogccldopt-llvm

$(PROGRAMS_TO_TEST:%=Output/%.nogccldopt-llvm): \
Output/%.nogccldopt-llvm: Output/%.linked.rbc $(LGCCLDPROG)
	-$(LGCCLD) -disable-opt $(STATS) $< -lc $(LIBS) -lcrtend -o Output/$*.nogccldopt-llvm

endif   # ifndef DISABLE_FOR_LLVM_PROGRAMS

# Targets to get the pass arguments that gccas and gccld are using...
Output/gccas-pass-args: $(LGCCAS) Output/.dir
	-$(LGCCAS) /dev/null -o /dev/null -debug-pass=Arguments > $@.1 2>&1
	sed 's/Pass Arguments: //' < $@.1 > $@
Output/gccld-pass-args: $(LGCCLDPROG) Output/.dir
	$(LLVMAS) < /dev/null > Output/gccld.test.bc
	$(LGCCLD) Output/gccld.test.bc -o Output/gccld.test-out -debug-pass=Arguments > $@.1 2>&1
	sed 's/Pass Arguments: //' < $@.1 > $@

# If the program requires exception handling support, enable (potentially
# expensive) support for it.
ifdef REQUIRES_EH_SUPPORT
LLCFLAGS += -enable-correct-eh-support
endif

#
# Rules to compile the program for the C Back End
#
$(PROGRAMS_TO_TEST:%=Output/%.cbe.c): \
Output/%.cbe.c: Output/%.llvm.bc $(LLC)
	-$(LLC) $(LLCFLAGS) -march=c $< -o $@ -f

$(PROGRAMS_TO_TEST:%=Output/%.cbe): \
Output/%.cbe: Output/%.cbe.c
	-$(CC) $< $(LDFLAGS) $(CFLAGS) -fno-strict-aliasing -O2 -o $@

#
# Compile a linked program to machine code with LLC.
#
$(PROGRAMS_TO_TEST:%=Output/%.llc.s): \
Output/%.llc.s: Output/%.llvm.bc $(LLC)
	-$(LLC) $(LLCFLAGS) -f $< -o $@

$(PROGRAMS_TO_TEST:%=Output/%.llc-ls.s): \
Output/%.llc-ls.s: Output/%.llvm.bc $(LLC)
	-$(LLC) $(LLCFLAGS) -f -regalloc=iterativescan $< -o $@


## Alternative command to run llc remotely on another machine:
## This is very useful when debugging:
##	-rsh psmith "cd `/bin/pwd`; llc $(LLCFLAGS) -f $< -o $@"

# Assemble (and link) an LLVM-linked program using the system assembler...
#
$(PROGRAMS_TO_TEST:%=Output/%.llc): \
Output/%.llc: Output/%.llc.s
	-$(CC) $(CFLAGS) $< $(LLCLIBS) $(LDFLAGS) -o $@

$(PROGRAMS_TO_TEST:%=Output/%.llc-ls): \
Output/%.llc-ls: Output/%.llc-ls.s
	-$(CC) $(CFLAGS) $< $(LLCLIBS) $(LDFLAGS) -o $@


#
# Rules to execute the program
#

# EXTRA_LLI_OPTS is used by the nightly tester to add arguments to invocations
# of the JIT and LLI in order to get timing info and statistics.
EXTRA_LLI_OPTS = 

LLI_OPTS = -force-interpreter=true $(EXTRA_LLI_OPTS)
JIT_OPTS = -force-interpreter=false $(EXTRA_LLI_OPTS)


# If the program requires exception handling support, enable (potentially
# expensive) support for it.
ifdef REQUIRES_EH_SUPPORT
JIT_OPTS += -enable-correct-eh-support
endif

native: $(PROGRAMS_TO_TEST:%=Output/%.native)
stripped-bytecode:: $(PROGRAMS_TO_TEST:%=Output/%.llvm.stripped.bc)

ifndef PROGRAMS_HAVE_CUSTOM_RUN_RULES

# Rules to build the test output...
ifndef USE_PRECOMPILED_BYTECODE
$(PROGRAMS_TO_TEST:%=Output/%.out-nat): \
Output/%.out-nat: Output/%.native
	-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-lli): \
Output/%.out-lli: Output/%.llvm.bc $(LLI)
	-$(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) $(LLI_OPTS) $< $(RUN_OPTIONS)

$(PROGRAMS_TO_TEST:%=Output/%.out-jit): \
Output/%.out-jit: Output/%.llvm.bc $(LLI)
	-$(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) $(JIT_OPTS) $< $(RUN_OPTIONS)
ifdef PROGRAM_REQUIRED_TO_EXIT_OK
	@if test \! -f $@.exitok; then echo "TEST (jit): $* FAILED!"; rm -f $@; fi
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-jit-ls): \
Output/%.out-jit-ls: Output/%.llvm.bc $(LLI)
	-$(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) -regalloc=iterativescan $(JIT_OPTS) $< $(RUN_OPTIONS)
ifdef PROGRAM_REQUIRED_TO_EXIT_OK
	@if test \! -f $@.exitok; then echo "TEST (jit-ls): $* FAILED!"; rm -f $@; fi
endif

$(PROGRAMS_TO_TEST:%=Output/%.out-llc): \
Output/%.out-llc: Output/%.llc
	-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)

$(PROGRAMS_TO_TEST:%=Output/%.out-llc-ls): \
Output/%.out-llc-ls: Output/%.llc-ls
	-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)

$(PROGRAMS_TO_TEST:%=Output/%.out-cbe): \
Output/%.out-cbe: Output/%.cbe
	-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)


# Specify stdin, reference output, and command line options for the program...
BUGPOINT_OPTIONS += -input=$(STDIN_FILENAME) -output=Output/$*.out-nat
BUGPOINT_OPTIONS += -timeout=$(RUNTIMELIMIT)
BUGPOINT_OPTIONS += --tool-args $(LLCFLAGS)
BUGPOINT_ARGS += --args -- $(RUN_OPTIONS)

# Rules to bugpoint the GCCAS, GCCLD, LLC, or LLI commands...
$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-gccas): \
Output/%.bugpoint-gccas: Output/%.noopt-llvm.bc $(LBUGPOINT) \
                         Output/gccas-pass-args Output/%.out-nat
	$(LBUGPOINT) $< `cat Output/gccas-pass-args` $(BUGPOINT_OPTIONS) $(BUGPOINT_ARGS)

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-gccld): \
Output/%.bugpoint-gccld: Output/%.nogccldopt-llvm.bc $(LBUGPOINT) \
                         Output/gccld-pass-args Output/%.out-nat
	$(LBUGPOINT) $< `cat Output/gccld-pass-args` $(OPTPASSES) $(BUGPOINT_OPTIONS) $(BUGPOINT_ARGS)

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-llc): \
Output/%.bugpoint-llc: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(LBUGPOINT) $< -run-llc $(BUGPOINT_OPTIONS) $(BUGPOINT_ARGS)

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-llc-ls): \
Output/%.bugpoint-llc-ls: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(LBUGPOINT) $< -run-llc $(BUGPOINT_OPTIONS) -regalloc=iterativescan $(BUGPOINT_ARGS)

$(PROGRAMS_TO_TEST:%=Output/%.bugpoint-jit): \
Output/%.bugpoint-jit: Output/%.llvm.bc $(LBUGPOINT) Output/%.out-nat
	$(LBUGPOINT) $< -run-jit $(BUGPOINT_OPTIONS) $(BUGPOINT_ARGS)


LIBPROFILESO = $(LLVM_OBJ_ROOT)/Debug/lib/libprofile_rt.so

$(PROGRAMS_TO_TEST:%=Output/%.prof): \
Output/%.prof: Output/%.llvm-prof.bc Output/%.out-nat $(LIBPROFILESO)
	@rm -f $@
	-$(RUNSAFELY) $(STDIN_FILENAME) Output/$*.out-prof $(LLI) $(JIT_OPTS) \
            -fake-argv0 'Output/$*.llvm.bc' -load $(LIBPROFILESO) $< -llvmprof-output $@ $(RUN_OPTIONS)
	@cmp -s Output/$*.out-prof Output/$*.out-nat || \
		printf "***\n***\n*** WARNING: Output of profiled program (Output/$*.out-prof)\n*** doesn't match the output of the native program (Output/$*.out-nat)!\n***\n***\n";

endif


#
# Rules to generate profiling information
#
$(PROGRAMS_TO_TEST:%=Output/%.llvm-prof.bc): \
Output/%.llvm-prof.bc: Output/%.llvm.bc
	$(LOPT) -insert-edge-profiling $< -o $@ -f

$(PROGRAMS_TO_TEST:%=Output/%.printprof): \
Output/%.printprof: Output/%.llvm.bc Output/%.prof $(LPROF)
	$(LPROF) $< Output/$*.prof


#
# Rules to diff test output...
#
ifdef PROGRAM_IS_NONDETERMINISTIC
# If the program is non-deterministic, just touch $@
DIFFPROG = touch $@ \#
endif

$(PROGRAMS_TO_TEST:%=Output/%.diff-lli): \
Output/%.diff-lli: Output/%.out-nat Output/%.out-lli
	-$(DIFFPROG) lli $* $(HIDEDIFF)

$(PROGRAMS_TO_TEST:%=Output/%.diff-jit): \
Output/%.diff-jit: Output/%.out-nat Output/%.out-jit
	-$(DIFFPROG) jit $* $(HIDEDIFF)

$(PROGRAMS_TO_TEST:%=Output/%.diff-jit-ls): \
Output/%.diff-jit-ls: Output/%.out-nat Output/%.out-jit-ls
	-$(DIFFPROG) jit-ls $* $(HIDEDIFF)

$(PROGRAMS_TO_TEST:%=Output/%.diff-llc): \
Output/%.diff-llc: Output/%.out-nat Output/%.out-llc
	-$(DIFFPROG) llc $* $(HIDEDIFF)

$(PROGRAMS_TO_TEST:%=Output/%.diff-llc-ls): \
Output/%.diff-llc-ls: Output/%.out-nat Output/%.out-llc-ls
	-$(DIFFPROG) llc-ls $* $(HIDEDIFF)

$(PROGRAMS_TO_TEST:%=Output/%.diff-cbe): \
Output/%.diff-cbe: Output/%.out-nat Output/%.out-cbe
	-$(DIFFPROG) cbe $* $(HIDEDIFF)

ifndef DISABLE_DIFFS
$(PROGRAMS_TO_TEST:%=Output/%.exe-lli): \
Output/%.exe-lli: Output/%.diff-lli
	-cp $< $@

$(PROGRAMS_TO_TEST:%=Output/%.exe-jit): \
Output/%.exe-jit: Output/%.diff-jit
	-cp $< $@

$(PROGRAMS_TO_TEST:%=Output/%.exe-jit-ls): \
Output/%.exe-jit-ls: Output/%.diff-jit-ls
	-cp $< $@

$(PROGRAMS_TO_TEST:%=Output/%.exe-llc): \
Output/%.exe-llc: Output/%.diff-llc
	-cp $< $@

$(PROGRAMS_TO_TEST:%=Output/%.exe-llc-ls): \
Output/%.exe-llc-ls: Output/%.diff-llc-ls
	-cp $< $@

$(PROGRAMS_TO_TEST:%=Output/%.exe-cbe): \
Output/%.exe-cbe: Output/%.diff-cbe
	-cp $< $@

# Pseudo target to build just the bytecode file.
bytecode:: $(PROGRAMS_TO_TEST:%=Output/%.llvm.bc)

endif

# Rules to support the USE_PRECOMPILED_BYTECODE setting If set, submakefiles
# will not know how to make output bytecode files for the programs in this
# directory.  Instead, this makefile just copies them out of the bytecode
# repository.
ifdef USE_PRECOMPILED_BYTECODE

# Calculate the directory we should copy the bytecode file into.  This is
# relative to BYTECODE_REPOSITORY and the current directory this program is in.
#
CURDIR  := $(shell cd .; pwd)
PROGDIR := $(shell cd $(LEVEL); pwd)
SRCDIR  := $(BYTECODE_REPOSITORY)/$(subst $(PROGDIR),,$(CURDIR))

# Because we don't have source code, we cannot build a native version of the
# executable.  Copy over pregenerated reference output.
$(PROGRAMS_TO_TEST:%=Output/%.out-nat): \
Output/%.out-nat: $(SRCDIR)/%.reference_output Output/.dir
	cp $< $@

# To make a raw bytecode file up-to-date, just copy it over.
$(PROGRAMS_TO_TEST:%=Output/%.linked.rll): \
Output/%.linked.rll: $(SRCDIR)/%.linked.rll Output/.dir
	cp $< $@

$(PROGRAMS_TO_TEST:%=Output/%.LOC.txt): \
Output/%.LOC.txt: $(SRCDIR)/%.LOC.txt Output/.dir
	cp $< $@
endif

# Support for the TEST= option... when TEST= is specified on the command line,
# the default target is the test target.  Here we dispatch to a specific set of
# tests.
#
.PHONY: test-setup test-finish
test-setup::
test-finish::
test:: test-setup $(PROGRAMS_TO_TEST:%=test.$(TEST).%) test-finish

# AVAILABLE_TESTS - Compute the set of tests available for user help
#
TEST_FILES = $(wildcard $(PROGDIR)/TEST.*.Makefile) \
             $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.*.Makefile)
AVAILABLE_TESTS = $(patsubst TEST.%.Makefile,%,$(notdir $(TEST_FILES)))

# If they just say 'make test' then we print out an error telling the user to
# specify a TEST= option.
$(PROGRAMS_TO_TEST:%=test..%): \
test..%:
	@echo
	@echo "***************************************************************"
	@echo "  ERROR: you cannot type '$(MAKE) test' directly."
	@echo "  Instead, use '$(MAKE) TEST=X' where X is the name of a test."
	@echo "  Tests available: $(AVAILABLE_TESTS)"
	@echo "  Alternatively, just use '$(MAKE)' to run comparisons."
	@echo "***************************************************************"
	@echo
	@exit 1

# Include all makefiles which define tests... These makefiles must define
# test.<testname>.%  given input from Output/%.llvm.bc
#
ifdef TEST
TestMakefile := $(wildcard $(PROGDIR)/TEST.$(TEST).Makefile) \
                $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.$(TEST).Makefile)
TestReport   := $(wildcard $(PROGDIR)/TEST.$(TEST).report) \
                $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.$(TEST).report)
TestGnuPlot  := $(wildcard $(PROGDIR)/TEST.$(TEST).gnuplot) \
                $(wildcard $(LLVM_SRC_ROOT)/projects/*/test/TEST.$(TEST).gnuplot)
ifneq ($(strip $(TestMakefile)),)
-include $(TestMakefile)
else
$(PROGRAMS_TO_TEST:%=test.$(TEST).%): \
test.$(TEST).%:
	@echo
	@echo "***************************************************************"
	@echo "  ERROR: Test '$(TEST)' is not a known test!"
	@echo "  Tests Available: $(AVAILABLE_TESTS)"
	@echo "  Test Program: $*"
	@echo "  Test Makefile: $(TestMakefile)"
	@echo "  Test Report: $(TestReport)"
	@echo "***************************************************************"
	@echo
	@exit 1
endif
endif

#
# Rules for building a report from 'make report TEST=<x>'
#
GENERATEREPORT := $(BUILD_SRC_ROOT)/GenerateReport.pl

report.$(TEST).raw.out: $(REPORT_DEPENDENCIES) $(TestMakefile)
	gmake TEST=$(TEST) 2>&1 | tee $@

ifneq ($(TestReport),)
report.$(TEST).txt: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
	$(GENERATEREPORT) $(TestReport) < $< > $@

report.$(TEST).html: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
	$(GENERATEREPORT) -html $(TestReport) < $< > $@

report.$(TEST).tex: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
	$(GENERATEREPORT) -latex $(TestReport) < $< > $@

report.$(TEST).csv: report.$(TEST).raw.out $(TestReport) $(GENERATEREPORT)
	$(GENERATEREPORT) -csv $(TestReport) < $< > $@

report.graphs: report.$(TEST).raw.out $(TestReport) $(TestGnuPlot) $(GENERATEREPORT)
	$(GENERATEREPORT) -graphs $(TestReport) < $<
	gnuplot $(TestGnuPlot)

report: report.$(TEST).txt
	@cat $<

report.html: report.$(TEST).html

report.csv: report.$(TEST).csv

report.tex: report.$(TEST).tex
	@cat $<
endif

clean::
	rm -f report.*.raw.out report.*.txt report.*.html report.*.tex report.*.csv