|
0
|
1 #!/usr/bin/perl
|
|
|
2
|
|
|
3 # Three-way DejaGNU comparison; uses dglib.pm. Run perldoc on this file for
|
|
|
4 # usage.
|
|
|
5 #
|
|
|
6 # Author: Matthew Sachs <msachs@apple.com>
|
|
|
7 #
|
|
|
8 # Copyright (c) 2006 Free Software Foundation.
|
|
|
9 #
|
|
|
10 # This file is part of GCC.
|
|
|
11 #
|
|
|
12 # GCC is free software; you can redistribute it and/or modify
|
|
|
13 # it under the terms of the GNU General Public License as published by
|
|
|
14 # the Free Software Foundation; either version 3, or (at your option)
|
|
|
15 # any later version.
|
|
|
16 #
|
|
|
17 # GCC is distributed in the hope that it will be useful,
|
|
|
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
20 # GNU General Public License for more details.
|
|
|
21 #
|
|
|
22 # You should have received a copy of the GNU General Public License
|
|
|
23 # along with GCC; see the file COPYING. If not, write to
|
|
|
24 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
|
|
|
25 # Boston, MA 02110-1301, USA.
|
|
|
26
|
|
|
27 =pod
|
|
|
28
|
|
|
29 =head1 SYNOPSIS
|
|
|
30
|
|
|
31 compareSumTests3 -- Two-way or three-way compare between DejaGNU .sum files
|
|
|
32
|
|
|
33 =head1 USAGE
|
|
|
34
|
|
|
35 compareSumTests3 old1.sum [old2.sum] new.sum
|
|
|
36 compareSumTests3 -i 1:2 -x 2:3 old1.sum old2.sum new.sum
|
|
|
37
|
|
|
38 =head1 DESCRIPTION
|
|
|
39
|
|
|
40 Gives results in terms of 'new' (e.g. things that work in 'new' and don't in
|
|
|
41 other compilers are improvements, things that don't in 'new' and do in others
|
|
|
42 are regressions, and it tells you which of the two old compilers (or both)
|
|
|
43 the test is a regression from.
|
|
|
44
|
|
|
45 We treat any DG result other than PASS or XFAIL as a failure, e.g.
|
|
|
46 UNRESOLVED, UNTESTED or test was not run.
|
|
|
47
|
|
|
48 We merge some tests into 'logical tests' with multiple subphases.
|
|
|
49 For instance, some tests will have compile, execute, and link
|
|
|
50 subtests. For these tests, if one of the phases fails, we
|
|
|
51 indicate which phase the failure originates in. For instance,
|
|
|
52 in the following test results:
|
|
|
53
|
|
|
54 gcc.c-torture/compile_execute/xxxx.c: [FAIL:C,FAIL:X,PASS]
|
|
|
55
|
|
|
56 the "compile_execute" replaces the compile or execute portion of the test name,
|
|
|
57 and "FAIL:C" and "FAIL:X" indicates where the combined test failed.
|
|
|
58
|
|
|
59 =head1 OPTIONS
|
|
|
60
|
|
|
61 =head2 OVERVIEW
|
|
|
62
|
|
|
63 =over 4
|
|
|
64
|
|
|
65 =item *
|
|
|
66
|
|
|
67 C<-i X:Y>: Only display differences between the two indicated runs.
|
|
|
68
|
|
|
69 =item *
|
|
|
70
|
|
|
71 C<-p>: Give plain output, suitable for piping to another program.
|
|
|
72
|
|
|
73 =item *
|
|
|
74
|
|
|
75 C<-x X:Y>: Exclude differences between the two indicated runs.
|
|
|
76
|
|
|
77 =back
|
|
|
78
|
|
|
79 =head2 PLAIN OUTPUT FORMAT
|
|
|
80
|
|
|
81 In the plain
|
|
|
82 output format, the category headers are not displayed and there are no tabs
|
|
|
83 in front of each result line. Instead, each result line has two characters
|
|
|
84 followed by a space in front of it. The first character will be either an 'I'
|
|
|
85 for improvement or 'R' for regression; the second character will be a 1, 2, or 3,
|
|
|
86 indicating which run was the odd one out.
|
|
|
87
|
|
|
88 =head2 SELECTING CHANGE SUBSETS
|
|
|
89
|
|
|
90 The following options cause only a selected subset of changes to be displayed.
|
|
|
91 These options ask for a "run", a number which is used to select
|
|
|
92 one of the three runs (C<old1>, C<old2>, or C<new>.) C<1> and C<2> signify C<old1> and C<old2>
|
|
|
93 respectively; 3 signifies C<new>. If multiple options are given, the changes displayed
|
|
|
94 will be those which obey all of the given restrictions.
|
|
|
95
|
|
|
96 Typical usage of these options is to express something like "give me all changes
|
|
|
97 between 2 and 3, except for those where there was the same difference betwen 1 and 2
|
|
|
98 (as between 2 and 3.)" This would be given as:
|
|
|
99
|
|
|
100 -i 2:3 -x 1:2
|
|
|
101
|
|
|
102 =over 4
|
|
|
103
|
|
|
104 =item *
|
|
|
105
|
|
|
106 C<-i X:Y>: Only differences which are present between the two runs given
|
|
|
107 are displayed. For instance, if C<-i 1:2> is given and test A passes in
|
|
|
108 runs 1 and 2 but fails in run 3, that result will not be displayed.
|
|
|
109
|
|
|
110 =item *
|
|
|
111
|
|
|
112 C<-x X:Y>: Differences which are identical to a difference between the two runs
|
|
|
113 given will B<not> be displayed. For instance, if C<-x 1:2> is given and
|
|
|
114 test A passes in run 1 and fails in runs 2 and 3, that result will not be
|
|
|
115 displayed (since C<-x> will cause the difference between 1 and 2 to be ignored,
|
|
|
116 and the difference in 1 and 3 parallels the difference between 1 and 2.)
|
|
|
117 This option may only be used in conjunction with C<-i>.
|
|
|
118
|
|
|
119 =back
|
|
|
120
|
|
|
121 =cut
|
|
|
122
|
|
|
123 use strict;
|
|
|
124 use warnings;
|
|
|
125 use Getopt::Long;
|
|
|
126
|
|
|
127 use FindBin qw($Bin);
|
|
|
128 use lib "$Bin";
|
|
|
129 use dglib;
|
|
|
130
|
|
|
131 my %options;
|
|
|
132 my $error = undef;
|
|
|
133
|
|
|
134 if(!GetOptions(
|
|
|
135 "p" => \$options{p},
|
|
|
136 "i=s" => \$options{i},
|
|
|
137 "x=s" => \$options{x},
|
|
|
138 )) {
|
|
|
139 $error = "";
|
|
|
140 } elsif(@ARGV != 2 and @ARGV != 3) {
|
|
|
141 $error = "";
|
|
|
142 } elsif($options{x} and !$options{i}) {
|
|
|
143 $error = "-x may only be given in conjunction with -i.";
|
|
|
144 } else {
|
|
|
145 foreach my $opt("i", "x") {
|
|
|
146 if($options{$opt} and
|
|
|
147 ($options{$opt} !~ /^([123]):([123])$/ or
|
|
|
148 $1 == $2)
|
|
|
149 ) {
|
|
|
150 $error = "Invalid -$opt argument.";
|
|
|
151 }
|
|
|
152 }
|
|
|
153 }
|
|
|
154
|
|
|
155 if(defined($error)) {
|
|
|
156 print STDERR "$error\n" if $error;
|
|
|
157 print STDERR "Usage: compareSumTests3 [-p] [-i X:Y [-x X:Y]] old1.sum old2.sum new.sum\n";
|
|
|
158 print STDERR "Try 'perldoc $0' for further information.\n";
|
|
|
159 exit 1;
|
|
|
160 }
|
|
|
161
|
|
|
162 my(@sumfiles) = @ARGV;
|
|
|
163 -f $_ || die "$_ is not a regular file!\n" foreach @sumfiles;
|
|
|
164 my(%results, @inc_changes, @exc_changes, %checksums);
|
|
|
165
|
|
|
166 # We decrement the values given so that they correspond
|
|
|
167 # to indices into our results array.
|
|
|
168 if($options{i}) {
|
|
|
169 $options{i} =~ /(\d+):(\d+)/;
|
|
|
170 @inc_changes = ($1 - 1, $2 - 1);
|
|
|
171 }
|
|
|
172 if($options{x}) {
|
|
|
173 $options{x} =~ /(\d+):(\d+)/;
|
|
|
174 @exc_changes = ($1 - 1, $2 - 1);
|
|
|
175 }
|
|
|
176
|
|
|
177
|
|
|
178 my %analyzed_results = compareSumFiles(\@sumfiles);
|
|
|
179
|
|
|
180 foreach my $cat (qw(improvements regressions miscellaneous)) {
|
|
|
181 if(@sumfiles == 3) {
|
|
|
182 my @subcounts;
|
|
|
183 if(!$options{p}) {
|
|
|
184 $subcounts[$_] = @{$analyzed_results{$cat}->[$_] || []} for(0..2);
|
|
|
185 print "\u$cat: ", ($subcounts[0]+$subcounts[1]+$subcounts[2]), "\n";
|
|
|
186 }
|
|
|
187
|
|
|
188 for(my $i = 0; $i < 3; $i++) {
|
|
|
189 if(!$options{p} and $cat ne "miscellaneous") {
|
|
|
190 if($i == 0) {
|
|
|
191 if($cat eq "regressions") {
|
|
|
192 print "\tSuccess in old1 only: $subcounts[$i]\n";
|
|
|
193 } else {
|
|
|
194 print "\tFailure in old1 only: $subcounts[$i]\n";
|
|
|
195 }
|
|
|
196 } elsif($i == 1) {
|
|
|
197 if($cat eq "regressions") {
|
|
|
198 print "\tSuccess in old2 only: $subcounts[$i]\n";
|
|
|
199 } else {
|
|
|
200 print "\tFailure in old2 only: $subcounts[$i]\n";
|
|
|
201 }
|
|
|
202 } else {
|
|
|
203 if($cat eq "regressions") {
|
|
|
204 print "\tFailure in new only: $subcounts[$i]\n";
|
|
|
205 } else {
|
|
|
206 print "\tSuccess in new only: $subcounts[$i]\n";
|
|
|
207 }
|
|
|
208 }
|
|
|
209 }
|
|
|
210
|
|
|
211 foreach my $test (sort {$a->{name} cmp $b->{name}} @{$analyzed_results{$cat}->[$i] || []}) {
|
|
|
212 if(!$options{p}) {
|
|
|
213 if($cat eq "miscellaneous") {
|
|
|
214 print "\t";
|
|
|
215 } else {
|
|
|
216 print "\t\t";
|
|
|
217 }
|
|
|
218 } else {
|
|
|
219 if($cat eq "regressions") {
|
|
|
220 print "R";
|
|
|
221 } else {
|
|
|
222 print "I";
|
|
|
223 }
|
|
|
224
|
|
|
225 print $i+1, " ";
|
|
|
226 }
|
|
|
227 printf "%s [%s,%s,%s]\n", $test->{name}, $test->{data}->[0], $test->{data}->[1], $test->{data}->[2];
|
|
|
228 }
|
|
|
229 }
|
|
|
230 } else {
|
|
|
231 if(!$options{p}) {
|
|
|
232 my $subcount = @{$analyzed_results{$cat}};
|
|
|
233 print "\u$cat: $subcount\n";
|
|
|
234 }
|
|
|
235
|
|
|
236 foreach my $test (sort {$a->{name} cmp $b->{name}} @{$analyzed_results{$cat}}) {
|
|
|
237 if(!$options{p}) {
|
|
|
238 print "\t";
|
|
|
239 } else {
|
|
|
240 if($cat eq "regressions") {
|
|
|
241 print "R"; } else {
|
|
|
242 print "I";
|
|
|
243 }
|
|
|
244
|
|
|
245 print " ";
|
|
|
246 }
|
|
|
247 printf "%s [%s,%s]\n", $test->{name}, $test->{data}->[0], $test->{data}->[1], $test->{data}->[2];
|
|
|
248 }
|
|
|
249 }
|
|
|
250 }
|
