| ##===- safecode/test/Makefile ------------------------------*- Makefile -*-===## |
| # |
| # Define the default test rule, which allows people to just type 'make test' in |
| # this directory. |
| # |
| ##===----------------------------------------------------------------------===## |
| |
| # The default target in this directory is the test:: target |
| test:: |
| |
| LEVEL = .. |
| include $(LEVEL)/Makefile.common |
| |
| ifndef NO_STABLE_NUMBERS |
| STABLERUN := GET_STABLE_NUMBERS=1 |
| else |
| STABLERUN := |
| endif |
| |
| ifndef NO_LARGE_SIZE |
| LARGESIZE := LARGE_PROBLEM_SIZE=1 |
| else |
| LARGESIZE := |
| endif |
| |
| ifdef ENABLE_OPTIMIZED |
| OPTIMIZED := ENABLE_OPTIMIZED=1 |
| else |
| OPTIMIZED := |
| endif |
| |
| RUNTIMELIMIT=3600 |
| |
| PROFILING_DAT_DIR := $(shell pwd)/data |
| PROFILING_RESULT_DIR := $(shell pwd)/data/profiling |
| |
| LARGE_PROBLEM_SIZE_DIRS := |
| |
| |
| LARGE_PROBLEM_SIZE_DIRS := \ |
| $(addprefix $(TEST_OBJDIR)/,$(LARGE_PROBLEM_SIZE_DIRS)) |
| |
| #DEBUG_SAFECODE := DEBUG_SAFECODE=1 |
| |
| NORMAL_PROBLEM_SIZE_DIRS := \ |
| MultiSource/Benchmarks/Olden \ |
| External/SPEC/CINT2000 \ |
| External/SPEC/CINT2006 \ |
| #MultiSource/Benchmarks/Olden/bisort \ |
| #External/SPEC/CINT2000/197.parser \ |
| #External/SPEC/CINT2000/164.gzip \ |
| #External/SPEC/CINT2000/256.bzip2 \ |
| #External/Povray \ |
| #MultiSource/Benchmarks/Olden/tsp \ |
| #SingleSource/Benchmarks \ |
| #MultiSource/Benchmarks/Olden/bh \ |
| #MultiSource/Benchmarks/Olden/em3d \ |
| #External/SPEC/CINT2000/181.mcf \ |
| #External/SPEC/CINT2000/254.gap \ |
| #External/SPEC/CINT2000/300.twolf \ |
| #External/SPEC/CINT2000/175.vpr \ |
| #External/SPEC/CINT2000/186.crafty \ |
| #MultiSource/Benchmarks/Olden/mst \ |
| #External/SPEC/CINT2000/197.parser \ |
| #MultiSource/Benchmarks/Olden/power \ |
| #External/SPEC/CINT2000/253.perlbmk \ |
| #External/SPEC/CINT2000/252.eon \ |
| #External/SPEC/CINT2000/186.crafty \ |
| #External/SPEC/CINT2000/181.mcf \ |
| #External/SPEC/CINT2000/164.gzip \ |
| #External/SPEC/CINT2000/255.vortex \ |
| #External/SPEC/CINT2000/175.vpr \ |
| #External/SPEC/CINT2000/186.crafty \ |
| #External/SPEC/CINT2000/254.gap \ |
| ## WORKING: \ |
| # \ |
| #External/SPEC/CINT2000/164.gzip \ |
| #External/SPEC/CINT2000/197.parser \ |
| ## FULL LIST: \ |
| # \ |
| #MultiSource/Benchmarks/Olden \ |
| #External/SPEC/CINT2000 \ |
| #External/SPEC/CINT2000/181.mcf \ |
| #MultiSource/Benchmarks/Olden/voronoi \ |
| #External/SPEC/CINT2000/176.gcc \ |
| #External/Namd \ |
| #External/SPEC/CINT2000/175.vpr \ |
| #External/SPEC/CINT2000 \ |
| #External/SPEC/CINT2000/252.eon \ |
| #MultiSource/Applications/Burg \ |
| #MultiSource/Benchmarks/Olden/health \ |
| #MultiSource/Benchmarks/Olden/perimeter \ |
| #MultiSource/Benchmarks/Olden/treeadd \ |
| # MultiSource/Benchmarks/Olden \ |
| # MultiSource/Benchmarks/Olden/em3d \ |
| # MultiSource/Benchmarks/Olden \ |
| # MultiSource/Benchmarks/Ptrdist \ |
| # SingleSource \ |
| # External/SPEC/CINT2000/300.twolf \ |
| # SingleSource \ |
| # MultiSource \ |
| # MultiSource/Benchmarks \ |
| # External/SPEC/CINT2000 \ |
| #External/SPEC/CFP2000/188.ammp \ |
| #MultiSource/Applications/ghttpd \ |
| #MultiSource/Benchmarks/Olden/bh \ |
| #MultiSource/Benchmarks/Ptrdist \ |
| #MultiSource/Applications \ |
| #MultiSource/Benchmarks |
| |
| |
| NORMAL_PROBLEM_SIZE_DIRS := \ |
| $(addprefix $(TEST_OBJDIR)/,$(NORMAL_PROBLEM_SIZE_DIRS)) |
| |
| ############################################################################## |
| # Target for preparing the test suite for a clean run |
| ############################################################################## |
| prepare: |
| (cd $(TEST_OBJDIR); $(MAKE) clean) |
| |
| ############################################################################## |
| # Targets for gathering statistics for programs for papers |
| ############################################################################## |
| |
| # Program tests for Parallel Checking |
| partest:: |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=par \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| $(LARGESIZE) $(STABLERUN) $(OPTIMIZED) \ |
| report.html report.csv; \ |
| mkdir -p $(PROFILING_RESULT_DIR)/`basename $$dir`; \ |
| mv $(PROFILING_DAT_DIR)/profiler.*.dat $(PROFILING_RESULT_DIR)/`basename $$dir` || true;) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s -j1 TEST=par \ |
| report) \ |
| done |
| |
| # Program tests for Pool Allocation |
| progtest:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=safecode \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| LARGE_PROBLEM_SIZE=1 \ |
| report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=safecode \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| $(LARGESIZE) $(STABLERUN) $(OPTIMIZED) \ |
| report.html report.csv; \ |
| mkdir -p $(PROFILING_RESULT_DIR)/`basename $$dir`; \ |
| mv $(PROFILING_DAT_DIR)/profiler.*.dat $(PROFILING_RESULT_DIR)/`basename $$dir` || true;) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=safecode \ |
| LARGE_PROBLEM_SIZE=1 report) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=safecode \ |
| report) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| # Program tests for SAFECode debug tool |
| progdebug:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| USE_LTO=$(USE_LTO) \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=debug \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| LARGE_PROBLEM_SIZE=1 \ |
| report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| USE_LTO=$(USE_LTO) \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=debug \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) $(DEBUG_SAFECODE) \ |
| $(LARGESIZE) $(STABLERUN) $(OPTIMIZED) \ |
| report.html report.csv; \ |
| ) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=debug \ |
| LARGE_PROBLEM_SIZE=1 report) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=debug \ |
| report) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| progbb:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=bb \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| LARGE_PROBLEM_SIZE=1 \ |
| report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=bb \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) $(DEBUG_SAFECODE) \ |
| $(LARGESIZE) $(STABLERUN) $(OPTIMIZED) \ |
| report.html report.csv; \ |
| ) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=bb \ |
| LARGE_PROBLEM_SIZE=1 report) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=bb \ |
| report) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| # Program tests for measuring all the different SAFECode overhead |
| progfull:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=full \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| LARGE_PROBLEM_SIZE=1 \ |
| report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=full \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| $(LARGESIZE) $(STABLERUN) $(OPTIMIZED) \ |
| report.html report.csv;) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=full \ |
| LARGE_PROBLEM_SIZE=1 report) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=full \ |
| report) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| progabc:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=abc \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) LARGE_PROBLEM_SIZE=1 report.html) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=abc \ |
| report.html) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=abc \ |
| LARGE_PROBLEM_SIZE=1 report) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=abc \ |
| report) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| # Run tests comparing SAFECode and Valgrind |
| progvg:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=valgrind \ |
| POOLALLOC_OBJDIR=$(POOLALLOC_OBJDIR) \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| LARGE_PROBLEM_SIZE=1\ |
| report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) TEST=valgrind \ |
| RUNTIMELIMIT=$(RUNTIMELIMIT) \ |
| report.html report.csv) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=valgrind \ |
| LARGE_PROBLEM_SIZE=1 report) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s TEST=valgrind \ |
| report) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| progperf:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=perf \ |
| LARGE_PROBLEM_SIZE=1 report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=perf \ |
| report.html report.csv) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s -j1 TEST=perf \ |
| LARGE_PROBLEM_SIZE=1 report report.csv) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s -j1 TEST=perf \ |
| report report.csv) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| progp4perf:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=p4perf \ |
| LARGE_PROBLEM_SIZE=1 report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=p4perf \ |
| report.html report.csv) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s -j1 TEST=p4perf \ |
| LARGE_PROBLEM_SIZE=1 report report.csv) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s -j1 TEST=p4perf \ |
| report report.csv) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| progcputrack:: |
| for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=cputrack \ |
| LARGE_PROBLEM_SIZE=1 report.html report.csv) \ |
| done |
| for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=cputrack \ |
| report.html report.csv) \ |
| done |
| @for dir in $(LARGE_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s -j1 TEST=cputrack \ |
| LARGE_PROBLEM_SIZE=1 report report.csv) \ |
| done |
| @for dir in $(NORMAL_PROBLEM_SIZE_DIRS); do \ |
| (cd $$dir; \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -s -j1 TEST=cputrack \ |
| report report.csv) \ |
| done |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| |
| ############################################################################## |
| # Targets for running tests and gathering statistics for arbitrary tests |
| ############################################################################## |
| |
| # test target - Descend into projects/test-suite and run the TEST.safecode.Makefile |
| # tests... |
| test:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=safecode \ |
| $(LARGESIZE) $(STABLERUN) report report.html) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| pacompiletime:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=pacompiletime \ |
| report report.html) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| ptrcomp:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=ptrcomp \ |
| $(LARGESIZE) $(STABLERUN) report report.html) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| ptrcomptest:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=ptrcomp \ |
| $(LARGESIZE) report) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| vtl:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=pavtl \ |
| test report report.csv) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| perf:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=perf \ |
| test report report.csv) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| optzn:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=optzn \ |
| $(LARGESIZE) $(STABLERUN) test report report.csv) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| p4perf:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=p4perf \ |
| test report report.csv) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| strace:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=strace \ |
| test) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| cputrack:: |
| (cd $(TEST_OBJDIR)/$(SUBDIR); \ |
| PROJECT_DIR=$(PROJ_OBJ_ROOT) $(MAKE) -j1 TEST=cputrack \ |
| report report.csv) |
| @printf "\a"; sleep 1; printf "\a"; sleep 1; printf "\a" |
| |
| ##===----------------------------------------------------------------------===## |
| # Lit tests |
| ##===----------------------------------------------------------------------===## |
| |
| .PHONY: lit litclean lit-core lit-cstdlib lit-formatstrings clean \ |
| lit-bodiagsuite |
| |
| # Path to SAFECode libraries |
| SC_LIB := $(PROJ_OBJ_ROOT)/$(BuildMode)/lib |
| SC_BIN := $(PROJ_OBJ_ROOT)/$(BuildMode)/bin/clang |
| |
| SETENV = TARGET_TRIPLE=$(TARGET_TRIPLE) \ |
| PYTHON=$(PYTHON) \ |
| PATH=$(PROJ_OBJ_ROOT)/test/tools:$(PATH) \ |
| SC_OBJ_ROOT=$(PROJ_OBJ_ROOT) |
| |
| # Path to test files |
| CORESRC=$(PROJ_SRC_ROOT)/test/core |
| CSTDLIB=$(PROJ_SRC_ROOT)/test/cstdlib |
| FMTSTR=$(PROJ_SRC_ROOT)/test/formatstrings |
| REGRSN=$(PROJ_SRC_ROOT)/test/regression |
| BODIAGSRC=$(PROJ_SRC_ROOT)/test/BOdiagsuite-20050808/testcases |
| |
| COREOBJ=$(PROJ_OBJ_ROOT)/test/cstdlib |
| CSTDLIBOBJ=$(PROJ_OBJ_ROOT)/test/cstdlib |
| FMTSTROBJ=$(PROJ_OBJ_ROOT)/test/formatstrings |
| REGRSNOBJ=$(PROJ_OBJ_ROOT)/test/regression |
| BODIAGOBJ=$(PROJ_OBJ_ROOT)/test/bodiagsuite |
| |
| TESTSCRIPT=$(PROJ_OBJ_ROOT)/test/tools/test.sh |
| |
| # ulimit settings |
| ULIMIT="ulimit -t 30 ; ulimit -d 512000 ; ulimit -m 512000 ; ulimit -v 1500000 ;" |
| |
| # Run all lit tests |
| lit: lit-core lit-formatstrings lit-cstdlib lit-regression lit-bodiagsuite |
| |
| # Run the lit tests for core SAFECode |
| lit-core: $(TESTSCRIPT) |
| @mkdir -p $(COREOBJ) |
| $(Verb) $(SETENV) $(MAKE) -C $(LLVM_OBJ_ROOT)/test check-local-lit TESTSUITE=$(CORESRC) LIT_ARGS=-j1 ULIMIT=$(ULIMIT) |
| |
| # Run the lit tests for cstdlib |
| lit-cstdlib: $(TESTSCRIPT) |
| @mkdir -p $(CSTDLIBOBJ) |
| $(Verb) $(SETENV) $(MAKE) -C $(LLVM_OBJ_ROOT)/test check-local-lit TESTSUITE=$(CSTDLIB) LIT_ARGS=-j1 ULIMIT=$(ULIMIT) |
| |
| # Run the lit tests for format string functions |
| lit-formatstrings: $(TESTSCRIPT) |
| @mkdir -p $(FMTSTROBJ) |
| $(Verb) $(SETENV) $(MAKE) -C $(LLVM_OBJ_ROOT)/test check-local-lit TESTSUITE=$(FMTSTR) LIT_ARGS=-j1 ULIMIT=$(ULIMIT) |
| |
| # Run regression tests |
| lit-regression: $(TESTSCRIPT) |
| @mkdir -p $(REGRSNOBJ) |
| $(Verb) $(SETENV) \ |
| PATH=$(PROJ_OBJ_ROOT)/$(BuildMode)/bin:$(LLVMToolDir):$(PATH) \ |
| $(MAKE) -C $(LLVM_OBJ_ROOT)/test check-local-lit TESTSUITE=$(REGRSN) ULIMIT=$(ULIMIT) |
| |
| # All names of the files in the BOdiagsuite |
| BODIAG_FILE_NAMES := $(notdir $(wildcard $(BODIAGSRC)/*.c)) |
| |
| # List of XFAIL'd tests; handled separately |
| BODIAG_XFAILS := $(shell awk '!/^\#/ { print $$1 }' $(BODIAGSRC)/../xfails) |
| |
| # The names of the files without errors in them |
| BODIAG_CORRECT_NAMES := \ |
| $(filter-out $(BODIAG_XFAILS),$(notdir $(wildcard $(BODIAGSRC)/*ok.c))) |
| |
| # The names of the files with errors in them |
| BODIAG_BAD_NAMES := \ |
| $(filter-out $(BODIAG_XFAILS) $(BODIAG_CORRECT_NAMES),$(BODIAG_FILE_NAMES)) |
| |
| # The paths to the test files annotated with lit run lines. |
| BODIAG_ANNOTATED := $(addprefix $(BODIAGOBJ)/,$(BODIAG_FILE_NAMES)) |
| |
| $(BODIAG_XFAILS:%=$(BODIAGOBJ)/%): $(BODIAGOBJ)/%: $(BODIAGSRC)/% |
| @mkdir -p $(BODIAGOBJ) |
| @printf > $@ '/*\n * RUN: test.sh -%s -t %%t %%s\n * XFAIL: %s\n */\n' \ |
| `if [[ $@ == *ok.c ]]; then printf p; else printf e; fi` \ |
| "`awk '!/^#/' $(BODIAGSRC)/../xfails | grep -F "$(notdir $<)" | cut -f2- -d' '`" |
| @cat $< >> $@ |
| |
| $(BODIAG_CORRECT_NAMES:%=$(BODIAGOBJ)/%): $(BODIAGOBJ)/%: $(BODIAGSRC)/% |
| @mkdir -p $(BODIAGOBJ) |
| @printf '/*\n * RUN: test.sh -p -t %%t %%s\n */\n' > $@ |
| @cat $< >> $@ |
| |
| $(BODIAG_BAD_NAMES:%=$(BODIAGOBJ)/%): $(BODIAGOBJ)/%: $(BODIAGSRC)/% |
| @mkdir -p $(BODIAGOBJ) |
| @printf '/*\n * RUN: test.sh -e -t %%t %%s\n */\n' > $@ |
| @cat $< >> $@ |
| |
| # Build the script to run the lit tests |
| $(TESTSCRIPT): $(PROJ_SRC_ROOT)/test/tools/test.sh.in |
| @echo Creating test script... |
| @mkdir -p `dirname $@` |
| @sed -e 's#@SC@#$(SC_BIN)#g' \ |
| -e 's#@SC_LIB@#$(SC_LIB)#g' < $< > $@ |
| chmod +x $@ |
| |
| # Run the BOdiagsuite |
| lit-bodiagsuite: $(TESTSCRIPT) $(BODIAG_ANNOTATED) |
| @cp $(BODIAGSRC)/TestInputFile1 $(BODIAGOBJ) |
| $(Verb) $(SETENV) $(MAKE) -C $(LLVM_OBJ_ROOT)/test check-local-lit \ |
| TESTSUITE=$(BODIAGSRC)/.. LIT_ARGS=-j2 ULIMIT=$(ULIMIT) |
| |
| clean:: litclean |
| |
| # Clean all files generated by the lit tests |
| litclean: |
| -rm -f $(PROJ_OBJ_ROOT)/test/tools/test.sh |
| -rm -rf $(PROJ_OBJ_ROOT)/test/core/Output |
| -rm -rf $(PROJ_OBJ_ROOT)/test/cstdlib/Output |
| -rm -rf $(PROJ_OBJ_ROOT)/test/formatstrings/Output |
| -rm -rf $(PROJ_OBJ_ROOT)/test/regression/Output |
| -rm -rf $(BODIAGOBJ)/Output |
