| #!/usr/bin/env perl |
| |
| # runtest [options] FILENAME |
| # |
| # Read the file FILENAME. Each line contains a test. |
| # Convert template to test and crosstest. |
| # If possilble generate orphaned testversions, too. |
| # Use make to compile the test |
| |
| ################################################################################ |
| # Global configuration options for the runtestscript itself: |
| ################################################################################ |
| |
| # name of the global configuration file for the testsuite: |
| $config_file = "ompts.conf"; |
| $logfile = "ompts.log"; # overwriteable by value in config file |
| $env_set_threads_command = 'OMP_NUM_THREADS=%n; export OMP_NUM_THREADS;'; |
| $debug_mode = 0; |
| ################################################################################ |
| # After this line the script part begins! Do not edit anithing below |
| ################################################################################ |
| |
| |
| # Namespaces: |
| use Getopt::Long; |
| #use Unix::PID; |
| use Data::Dumper; |
| use ompts_parserFunctions; |
| |
| # Extracting given options |
| GetOptions("help", |
| "listlanguages", |
| "lang=s", |
| "list", |
| "testinfo=s", |
| "numthreads=i", |
| "test=s", |
| "compile!", |
| "run!", |
| "orphan!", |
| "resultfile=s" |
| ); |
| |
| # Get global configuratino options from config file: |
| if(! -e $config_file){ error ("Could not find config file $config_file\n", 1);} |
| open (CONFIG, "<$config_file") or error ("Could not open config file $config_file\n", 2); |
| while (<CONFIG>) { $config .= $_; } |
| close (CONFIG); |
| |
| ($logfile) = get_tag_values ("logfile", $config); |
| ($timeout) = get_tag_values ("singletesttimeout", $config); |
| ($display_errors) = get_tag_values("displayerrors", $config); |
| ($display_warnings) = get_tag_values ("displaywarnings", $config); |
| ($numthreads) = get_tag_values ("numthreads", $config); |
| ($env_set_threads_command) = get_tag_values("envsetthreadscommand",$config); |
| $env_set_threads_command =~ s/\%n/$numthreads/g; |
| @languages = get_tag_values ("language", $config); |
| |
| if (!defined($opt_compile)) {$opt_compile = 1;} |
| if (!defined($opt_run)) {$opt_run = 1;} |
| if (!defined($opt_orphan)) {$opt_orphan = 1;} |
| if (!defined($opt_resultsfile)) {($opt_resultsfile) = get_tag_values("resultsfile", $config);} |
| if ( defined($opt_numthreads) && ($opt_numthreads > 0)) {$numthreads = $opt_numthreads;} |
| if ($debug_mode) { |
| print <<EOF; |
| Testsuite configuration: |
| Logfile = $logfile |
| Timeout = $timeout seconds |
| Language: $opt_lang |
| Display errors: $display_errors |
| Display warnings: $display_warnings |
| Resultsfile: $opt_resultsfile |
| Numthreads: $numthreads |
| ------------------------------ |
| EOF |
| } |
| |
| $num_construcs = 0; |
| $num_tests = 0; |
| $num_failed_tests = 0; |
| $num_successful_tests = 0; |
| $num_verified_tests = 0; |
| $num_failed_compilation = 0; |
| |
| $num_normal_tests_failed = 0; |
| $num_normal_tests_compile_error = 0; |
| $num_normal_tests_timed_out = 0; |
| $num_normal_tests_successful = 0; |
| $num_normal_tests_verified = 0; |
| |
| $num_orphaned_tests_failed = 0; |
| $num_orphaned_tests_compile_error = 0; |
| $num_orphaned_tests_timed_out = 0; |
| $num_orphaned_tests_successful = 0; |
| $num_orphaned_tests_verified = 0; |
| |
| if ($opt_help) { print_help_text (); exit 0; } |
| if ($opt_listlanguages){ print_avail_langs (); exit 0; } |
| if ($opt_list) { print_avail_tests (); exit 0; } |
| if ($opt_testinfo) { print_testinfo (); exit 0; } |
| if ($opt_test) { write_result_file_head(); |
| execute_single_test (); exit 0; } |
| if (-e $ARGV[0]) { write_result_file_head(); |
| execute_testlist($ARGV[0]); print_results(); |
| result_summary(); exit 0;} |
| |
| ################################################################################ |
| # sub function definitions |
| ################################################################################ |
| |
| # Function which prints the results file |
| sub print_results |
| { |
| system("echo; cat $opt_resultsfile; echo;"); |
| } |
| |
| # Function which prints a summary of all test |
| sub result_summary |
| { |
| my $num_directives = @test_results; |
| |
| print <<EOF; |
| |
| Summary: |
| S Number of tested Open MP constructs: $num_constructs |
| S Number of used tests: $num_tests |
| S Number of failed tests: $num_failed_tests |
| S Number of successful tests: $num_successful_tests |
| S + from this were verified: $num_verified_tests |
| |
| Normal tests: |
| N Number of failed tests: $num_normal_tests_failed |
| N + from this fail compilation: $num_normal_tests_compile_error |
| N + from this timed out $num_normal_tests_timed_out |
| N Number of successful tests: $num_normal_tests_successful |
| N + from this were verified: $num_normal_tests_verified |
| |
| Orphaned tests: |
| O Number of failed tests: $num_orphaned_tests_failed |
| O + from this fail compilation: $num_orphaned_tests_compile_error |
| O + from this timed out $num_orphaned_tests_timed_out |
| O Number of successful tests: $num_orphaned_tests_successful |
| O + from this were verified: $num_orphaned_tests_verified |
| EOF |
| |
| } |
| |
| # Function that executest the tests specified in the given list |
| sub execute_testlist |
| { |
| my ($filename) = @_; |
| # opening testlist |
| open(TESTS,$filename) or error ("Could not open $filename\n", 1); |
| TEST: while (<TESTS>) { |
| if (/^\s*#/) {next TEST;} |
| if (/^\s*$/) {next TEST;} |
| $opt_test = $_; |
| chomp ($opt_test); |
| execute_single_test (); |
| } |
| # print Dumper(@test_results); |
| } |
| |
| # Function that executes a system command but takes care of the global timeout |
| # If command did not finish inbetween returns '-' otherwise the exit status of |
| # the system command |
| sub timed_sys_command |
| { |
| my ($command) = @_; |
| my $exit_status = '-'; |
| |
| # set up the timeout for the command |
| eval { |
| local $SIG{ALRM} = sub {die "alarm\n"}; |
| alarm $timeout; |
| log_message_add ("Starting command \"$command\""); |
| $exit_status = system ($command); |
| alarm 0; |
| }; |
| # check if command finished during the maximum execution time |
| if ($@ eq "alarm\n") { |
| # test timed out |
| # my $pid = Unix::PID->new(); |
| # $pid->get_pidof($command, 1); |
| # $pid->kill(); |
| if ($debug_mode) { |
| log_message_add ("Command \"$command\" reached max execution time.\n"); |
| } |
| return "TO"; |
| } |
| # test finished |
| return $exit_status; |
| } |
| |
| # Function that runs the tests given as a array containing the testnames |
| # Returns an array containing the percent values of the passed tests and the |
| # successful crosstests. |
| sub run_test |
| { |
| my ($testname, $orphan) = @_; |
| my $bin_name, $cbin_name; |
| my $cmd, $exit_status, $failed; |
| my $resulttest, $resultctest; |
| |
| # path to test and crosstest either in normal or in orphaned version |
| if ($orphan) { |
| $bin_name = "bin/$opt_lang/orph_test_$testname"; |
| $cbin_name = "bin/$opt_lang/orph_ctest_$testname"; |
| } else { |
| $bin_name = "bin/$opt_lang/test_$testname"; |
| $cbin_name = "bin/$opt_lang/ctest_$testname"; |
| } |
| # Check if executables exist |
| if (! -e $bin_name) { |
| test_error ("Could not find executable \"$bin_name\"."); |
| return ('test' => '-', 'crosstest' => '-'); |
| } |
| # run the test |
| $cmd = "$env_set_threads_command ./$bin_name >$bin_name.out"; |
| print "Running test with $numthreads threads ."; |
| $exit_status = timed_sys_command ($cmd); |
| ############################################################ |
| # Check if test finished within max execution time |
| if ($exit_status eq 'TO') { |
| print ".... failed (timeout)\n"; |
| return ('test' => 'TO', 'crosstest' => '-') |
| } |
| ############################################################ |
| # check if all tests were successful |
| $failed = $exit_status >> 8; |
| if ($failed < 0 or $failed > 100) { $failed = 100; } |
| $resulttest = 100 - $failed; |
| if ($resulttest eq 100) { |
| print ".... success ..."; |
| } else { |
| print ".... failed $failed\% of the tests\n"; |
| return ('test' => $resulttest, 'crosstest' => '-'); |
| } |
| ############################################################ |
| |
| # do crosstest |
| # check if executable exist |
| if (! -e $cbin_name) { |
| test_error ("Could not find executable \"$cbin_name\"."); |
| print "... not verified (crosstest missing)\n"; |
| return ('test' => $resulttest, 'crosstest' => '-'); |
| } |
| # run crosstest |
| # Test was successful, so it makes sense to run the crosstest |
| $cmd = "$env_set_threads_command ./$cbin_name > $cbin_name.out"; |
| $exit_status = timed_sys_command ($cmd); |
| ############################################################ |
| # Check if crosstest finished within max execution time |
| if ($exit_status eq 'TO') { |
| print "... not verified (timeout)\n"; |
| return ('test' => $result, 'crosstest' => 'TO'); |
| } |
| ############################################################ |
| # test if crosstests failed as expected |
| $resultctest = $exit_status >> 8; |
| if ($resultctest > 0) { |
| print "... and verified with $resultctest\% certainty\n"; |
| } else { |
| print "... but might be lucky\n"; |
| } |
| return ('test' => $resulttest, 'crosstest' => $resultctest); |
| ############################################################ |
| } |
| |
| # Function that generates the test binaries out of the sourcecode |
| sub compile_src |
| { |
| my ($testname, $orphan) = @_; |
| print "Compiling soures ............"; |
| if ($orphan) { |
| # Make orphaned tests |
| $exec_name = "bin/$opt_lang/orph_test_$testname"; |
| $crossexe_name = "bin/$opt_lang/orph_ctest_$testname"; |
| $resulttest = system ("make $exec_name > $exec_name\_compile.log" ); |
| $resultctest = system ("make $crossexe_name > $crossexe_name\_compile.log" ); |
| } else { |
| # Make test |
| $exec_name = "bin/$opt_lang/test_$testname"; |
| $crossexe_name = "bin/$opt_lang/ctest_$testname"; |
| $resulttest = system ("make $exec_name > $exec_name\_compile.log" ); |
| $resultctest = system ("make $crossexe_name > $crossexe_name\_compile.log" ); |
| } |
| if ($resulttest) { test_error ("Compilation of the test failed."); } |
| if ($resultctest){ test_error ("Compilation of the crosstest failed."); } |
| |
| if ($resulttest or $resultctest) { |
| print ".... failed\n"; |
| return 0; |
| } else { |
| print ".... success\n"; |
| return 1; |
| } |
| } |
| |
| # Function which prepare the directory structure: |
| sub init_directory_structure |
| { |
| my ($language) = @_; |
| if (-e "bin" && -d "bin") { warning ("Old binary directory detected!");} |
| else { system ("mkdir bin"); } |
| if (-e "bin/$language" && -d "bin/$language") { |
| warning ("Old binary directory for language $language found.");} |
| else { system ("mkdir bin/$language"); } |
| } |
| |
| # Function that generates the sourcecode for the given test |
| sub make_src |
| { |
| my ($testname, $orphan) = @_; |
| my $template_file; |
| my $src_name; |
| |
| $template_file = "$dir/$testname.$extension"; |
| if (!-e $template_file) { test_error ("Could not find template for \"$testname\""); } |
| |
| print "Generating sources .........."; |
| if ($orphan) { |
| # Make orphaned tests |
| $src_name = "bin/$opt_lang/orph_test_$testname.$extension"; |
| $resulttest = system ("./$templateparsername --test --orphan $template_file $src_name"); |
| $src_name = "bin/$opt_lang/orph_ctest_$testname.$extension"; |
| $resultctest = system ("./$templateparsername --crosstest --orphan $template_file $src_name"); |
| } else { |
| # Make test |
| $src_name = "bin/$opt_lang/test_$testname.$extension"; |
| $resulttest = system ("./$templateparsername --test --noorphan $template_file $src_name"); |
| $src_name = "bin/$opt_lang/ctest_$testname.$extension"; |
| $resultctest = system ("./$templateparsername --crosstest --noorphan $template_file $src_name"); |
| } |
| if ($resulttest) { test_error ("Generation of sourcecode for the test failed."); } |
| if ($resultctest){ test_error ("Generation of sourcecode for the crosstest failed."); } |
| |
| if ($resulttest or $resultctest) { |
| print ".... failed\n"; |
| return 0; |
| } else { |
| print ".... success\n"; |
| return 1; |
| } |
| } |
| |
| # Function which checks if a given test is orphanable |
| sub test_is_orphanable |
| { |
| my ($testname) = @_; |
| my $src; |
| my $file = "$dir/$testname.$extension"; |
| if(! -e $file){ test_error ("Could not find test file $file\n");} |
| open (TEST, "<$file") or test_error ("Could not open test file $file\n"); |
| while (<TEST>) { $src .= $_; } |
| close (TEST); |
| return $src =~/ompts:orphan/; |
| } |
| |
| sub write_result_file_head |
| { |
| open (RESULTS, ">$opt_resultsfile") or error ("Could not open file '$opt_resultsfile' to write results.", 1); |
| $resultline = sprintf "%-25s %-s\n", "#Tested Directive", "\tt\tct\tot\toct"; |
| print RESULTS $resultline; |
| } |
| |
| # Function which adds a result to the list of results |
| sub add_result |
| { |
| my ($testname, $result) = @_; |
| # print Dumper(@{$result}); |
| |
| $num_constructs++; |
| |
| open (RESULTS, ">>$opt_resultsfile") or error ("Could not open file '$opt_resultsfile' to write results.", 1); |
| |
| if (${$result}[0][0]) { |
| $num_tests ++;} |
| |
| if ($opt_compile and ${$result}[0][1] eq 0) { |
| ${$result}[0][2]{test} = 'ce'; |
| ${$result}[0][2]{crosstest} = '-'; |
| $num_normal_tests_compile_error++; |
| $num_normal_tests_failed++; |
| } |
| |
| if ($opt_run and ${$result}[0][2] and ${$result}[0][2]{test} ne 'ce') { |
| if (${$result}[0][2]{test} == 100) { |
| $num_normal_tests_successful++; |
| if (${$result}[0][2]{crosstest} == 100){ |
| $num_normal_tests_verified++;} |
| } elsif (${$result}[0][2]{test} eq 'TO'){ |
| $num_normal_tests_timed_out++; |
| $num_normal_tests_failed++; |
| } else { |
| $num_normal_tests_failed++; |
| } |
| } |
| $resultline = "${$result}[0][2]{test}\t${$result}[0][2]{crosstest}\t"; |
| |
| if (${$result}[1][0]) { |
| $num_tests ++;} |
| else { $resultline .= "-\t-\n"; } |
| |
| if ($opt_compile and ${$result}[1][1] eq 0) { |
| ${$result}[1][2]{test} = 'ce'; |
| ${$result}[1][2]{crosstest} = '-'; |
| $num_orphaned_tests_compile_error++; |
| $num_orphaned_tests_failed++; |
| } |
| |
| if ($opt_run and ${$result}[1][2] and ${$result}[1][2]{test} ne 'ce') { |
| if (${$result}[1][2]{test} == 100) { |
| $num_orphaned_tests_successful++; |
| if (${$result}[1][2]{crosstest} == 100){ |
| $num_orphaned_tests_verified++;} |
| } elsif (${$result}[1][2]{test} eq 'TO'){ |
| $num_orphaned_tests_timed_out++; |
| $num_orphaned_tests_failed++; |
| } else { |
| $num_orphaned_tests_failed++; |
| } |
| } |
| $resultline .= "${$result}[1][2]{test}\t${$result}[1][2]{crosstest}\n"; |
| |
| $num_failed_tests = $num_normal_tests_failed + $num_orphaned_tests_failed; |
| $num_failed_compilation = $num_normal_tests_compile_error + $num_orphaned_tests_compile_error; |
| $num_successful_tests = $num_normal_tests_successful + $num_orphaned_tests_successful; |
| $num_verified_tests = $num_normal_tests_verified + $num_orphaned_tests_verified; |
| |
| $resultline2 = sprintf "%-25s %-s", "$testname", "\t$resultline"; |
| print RESULTS $resultline2; |
| } |
| |
| # Function which executes a single test |
| sub execute_single_test |
| { |
| my @result; |
| init_language_settings ($opt_lang); |
| init_directory_structure ($opt_lang); |
| log_message_add ("Testing for \"$opt_test\":"); |
| print "Testing for \"$opt_test\":\n"; |
| # tests in normal mode |
| if ($opt_compile){ $result[0][0] = make_src ($opt_test, 0); |
| $result[0][1] = compile_src ($opt_test, 0);} |
| if ($opt_run && $result[0][1] == 1) { |
| $result[0][2] = {run_test ($opt_test, 0)};} |
| # tests in orphaned mode |
| if ($opt_orphan && test_is_orphanable($opt_test)){ |
| log_message_add ("Testing for \"$opt_test\" in orphaned mode:"); |
| print "+ orphaned mode:\n"; |
| if ($opt_compile) { $result[1][0] = make_src ($opt_test, 1); |
| $result[1][1] = compile_src ($opt_test, 1);} |
| if ($opt_run && $result[1][1] == 1) { |
| $result[1][2] = {run_test ($opt_test, 1)};} |
| } |
| add_result($opt_test, \@result); |
| } |
| |
| # Function that prints info about a given test |
| sub print_testinfo |
| { |
| init_language_settings($opt_lang); |
| my $doc = ""; |
| my $file = $dir."/".$opt_testinfo.".".$extension; |
| if (! -e $file) {error ("Could not find template for test $opt_testinfo", 5);} |
| open (TEST,"<$file") or error ("Could not open template file \"$file\" for test $opt_testinfo", 6); |
| while (<TEST>) {$doc .= $_;} |
| close (TEST); |
| |
| (my $omp_version) = get_tag_values ("ompts:ompversion", $doc); |
| (my $dependences) = get_tag_values ("ompts:dependences", $doc); |
| (my $description) = get_tag_values ("ompts:testdescription", $doc); |
| my $orphanable = 'no'; |
| if ($doc =~ /ompts:orphan/) {$orphanable = 'yes';} |
| print <<EOF; |
| Info for test $opt_testinfo: |
| Open MP standard: $omp_version |
| Orphaned mode: $orphanable |
| Dependencies: $dependences |
| Description: $description |
| EOF |
| } |
| |
| # Function that initializes the settings for the given language |
| sub init_language_settings |
| { |
| my ($language) = @_; |
| foreach my $lang (@languages) { |
| (my $name) = get_tag_values ("languagename", $lang); |
| if ($name eq $language) { |
| ($extension) = get_tag_values ("fileextension", $lang); |
| ($dir) = get_tag_values ("dir", $lang); |
| ($templateparsername) = get_tag_values ("templateparsername", $lang); |
| last; |
| } |
| } |
| # Check if we found the specified language in the config file |
| if (!$extension and !$dir) { |
| error ("Language $language could not be found.\n", 3); |
| } |
| } |
| |
| |
| |
| # Function that prints all available tests for the given language |
| sub print_avail_tests |
| { |
| init_language_settings($opt_lang); |
| my @tests; |
| opendir(DIR,"$dir") or error ("Could not open directory $dir", 4); |
| while($_ = readdir(DIR)) { if (/\.$extension$/) {s/\.$extension//; push (@tests, $_);}} |
| closedir(DIR); |
| print "Found ".(@tests)." tests:\n". "-" x 30 . "\n"; |
| foreach (@tests) { print $_."\n";} |
| } |
| |
| # Function that prints all available tests for the given language |
| sub print_avail_langs |
| { |
| if (@languages > 0) { |
| print "Available languages:\n"; |
| foreach (@languages) { |
| (my $name) = get_tag_values ("languagename", $_); |
| print "$name\n"; |
| } |
| } else { |
| print "No languages available\n"; |
| } |
| } |
| |
| # Function that prints the error message |
| sub print_help_text |
| { |
| print <<EOF; |
| runtest.pl [options] [FILE] |
| |
| Executes the tests listed in FILE. FILE has to contain the names of the tests, |
| one test per line. Lines starting with '#' will be ignored. |
| A language has to be specified for all commands except --help and --listlanguages. |
| |
| Options: |
| --help displays this help message |
| --listlanguages lists all available languages |
| --lang=s select language |
| --list list available tests for a language |
| --testinfo=NAME show info for test NAME |
| --numthreads=NUM set number of threads (overwrites config file settings) |
| --test=NAME execute single test NAME |
| --nocompile do not compile tests |
| --norun do not run tests |
| --noorphan switch of orphaned tests |
| --resultfile=NAME use NAME as resultfile (overwrites config file settings) |
| EOF |
| } |
| |
| # Function that writes an error message for a failed test / part of a test |
| sub test_error |
| { |
| my ($message) = @_; |
| log_message_add ("ERROR: $message"); |
| if ($display_errors eq 1) { print STDERR "ERROR: $message\n"; } |
| } |
| |
| # Function that returns an warning message |
| sub warning { |
| my ($message) = @_; |
| if ($display_warnings eq 1) { print "Warniong: $message\n"; } |
| log_message_add ("Warning: $message"); |
| } |
| |
| # Function that returns an error message and exits with the specified error code |
| sub error { |
| my ($message, $error_code) = @_; |
| if ($display_errors eq 1) { print STDERR "ERROR: $message\n"; } |
| log_message_add ("ERROR: $message"); |
| exit ($error_code); |
| } |
| |
| # Function which adds an new entry into the logfile together with a timestamp |
| sub log_message_add |
| { |
| (my $message) = @_; |
| ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst) = localtime(); |
| if(length($hour) == 1) { $hour="0$hour"; } |
| if(length($min) == 1) { $min="0$min"; } |
| if(length($sec) == 1) { $sec="0$sec"; } |
| $mon=$mon+1; |
| $year=$year+1900; |
| open (LOGFILE,">>$logfile") or die "ERROR: Could not create $logfile\n"; |
| print LOGFILE "$mday/$mon/$year $hour.$min.$sec: $message\n"; |
| } |
