WebKitTools/Scripts/check-dom-results

   1 #!/usr/bin/perl -w
   2
   3 # Copyright (C) 2005 Apple Computer, Inc.  All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions
   7 # are met:
   8 #
   9 # 1.  Redistributions of source code must retain the above copyright
  10 #     notice, this list of conditions and the following disclaimer.
  11 # 2.  Redistributions in binary form must reproduce the above copyright
  12 #     notice, this list of conditions and the following disclaimer in the
  13 #     documentation and/or other materials provided with the distribution.
  14 # 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
  15 #     its contributors may be used to endorse or promote products derived
  16 #     from this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
  19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
  22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  23 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  25 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 # Script to check status of W3C DOM tests that are part of the WebKit tests.
  30
  31 use strict;
  32 use FindBin;
  33 use Cwd;
  34 use lib $FindBin::Bin;
  35 use webkitdirs;
  36
  37 chdirWebKit();
  38
  39 my $verbose = $ARGV[0] && $ARGV[0] eq "-v";
  40
  41 my $workingDir = getcwd();
  42 my $testDirectory = "$workingDir/LayoutTests";
  43
  44 my @suites = ( {"name" => "DOM Level 1 Core   (html)", "directory" => "dom/html/level1/core"},
  45                {"name" => "DOM Level 2 Core   (html)", "directory" => "dom/html/level2/core"},
  46                {"name" => "DOM Level 2 Events (html)", "directory" => "dom/html/level2/events"},
  47                {"name" => "DOM Level 2 HTML   (html)", "directory" => "dom/html/level2/html"},
  48                {"name" => "DOM Level 1 Core   (xhtml)", "directory" => "dom/xhtml/level1/core"},
  49                {"name" => "DOM Level 2 Core   (xhtml)", "directory" => "dom/xhtml/level2/core"},
  50                {"name" => "DOM Level 2 Events (xhtml)", "directory" => "dom/xhtml/level2/events"},
  51                {"name" => "DOM Level 2 HTML   (xhtml)", "directory" => "dom/xhtml/level2/html"},
  52                {"name" => "DOM Level 3 Core   (xhtml)", "directory" => "dom/xhtml/level3/core"},
  53                {"name" => "DOM Level 3 XPath  (svg)", "directory" => "dom/svg/level3/xpath"});
  54
  55 my $totalCount = 0;
  56 my $totalSuccesses = 0;
  57 my $totalDisabled = 0;
  58 my $totalFailed = 0;
  59
  60 foreach my $suite (@suites) {
  61
  62     my %suite = %$suite;
  63     my $directory = $suite{"directory"};
  64     my $name = $suite{"name"};
  65     my @results = `find "${testDirectory}/${directory}" -name "*-expected.txt"`;
  66     my @disabled = `find "${testDirectory}/${directory}" -name "*-disabled"`;
  67
  68     my @failures = ();
  69     my $count = 0;
  70
  71     foreach my $result (@results) {
  72         $count++;
  73         my $success = 0;
  74         open RESULT, "<$result";
  75         while (<RESULT>) {
  76             if (/Success/) {
  77                 $success = 1;
  78                 last;
  79             }
  80         }
  81         close RESULT;
  82         if (!$success) {
  83             push @failures, $result;
  84         }
  85     }
  86
  87     my $disabledCount = (scalar @disabled);
  88     my $failureCount = (scalar @failures);
  89
  90     $count += $disabledCount;
  91
  92     my $successCount = $count - $failureCount - $disabledCount;
  93     my $percentage = (sprintf "%.1f", ($successCount * 100.0 / $count));
  94
  95     if ($percentage == 100) {
  96         print "${name}: all ${count} tests succeeded";
  97     } else {
  98         print "${name}: ${successCount} out of ${count} tests succeeded (${percentage}%)";
  99     }
 100     print " ($disabledCount disabled)" if $disabledCount;
 101     print "\n";
 102     if ($verbose) {
 103         print "\n";
 104         if (@disabled) {
 105             print "    Disabled:\n";
 106
 107             foreach my $failure (sort @disabled) {
 108                 $failure =~ s|.*/||;
 109                 $failure =~ s|-disabled||;
 110                 print "        ${directory}/${failure}";
 111             }
 112         }
 113         if (@failures) {
 114             print "    Failed:\n";
 115
 116             foreach my $failure (sort @failures) {
 117                 $directory =~ m|^dom/(\w+)|;
 118                 my $extension = $1;
 119                 $failure =~ s|.*/||;
 120                 $failure =~ s|-expected\.txt|.${extension}|;
 121                 print "        ${directory}/${failure}";
 122             }
 123         }
 124
 125         print "\n";
 126     }
 127
 128     $totalCount += $count;
 129     $totalSuccesses += $successCount;
 130     $totalDisabled += $disabledCount;
 131     $totalFailed += $failureCount;
 132 }
 133
 134
 135 my $totalPercentage = (sprintf "%.1f", ($totalSuccesses * 100.0 / $totalCount));
 136 my $totalDisabledPercentage = (sprintf "%.1f", ($totalDisabled * 100.0 / $totalCount));
 137 my $totalFailedPercentage = (sprintf "%.1f", ($totalFailed * 100.0 / $totalCount));
 138
 139 print "Total: ${totalSuccesses} out of ${totalCount} tests succeeded (${totalPercentage}%)\n";
 140 print "       ${totalDisabled} tests disabled (${totalDisabledPercentage}%)\n";
 141 print "       ${totalFailed} tests failed (${totalFailedPercentage}%)\n";