1 /*
   2  * $Id$
   3  *
   4  * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.  Oracle designates this
  10  * particular file as subject to the "Classpath" exception as provided
  11  * by Oracle in the LICENSE file that accompanied this code.
  12  *
  13  * This code is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  16  * version 2 for more details (a copy is included in the LICENSE file that
  17  * accompanied this code).
  18  *
  19  * You should have received a copy of the GNU General Public License version
  20  * 2 along with this work; if not, write to the Free Software Foundation,
  21  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  22  *
  23  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  24  * or visit www.oracle.com if you need additional information or have any
  25  * questions.
  26  */
  27 package com.sun.javatest.report;
  28 
  29 import com.sun.javatest.KnownFailuresList;
  30 import com.sun.javatest.Status;
  31 import com.sun.javatest.TestResult;
  32 import com.sun.javatest.TestResultTable;
  33 import com.sun.javatest.util.StringArray;
  34 import java.io.BufferedReader;
  35 import java.io.IOException;
  36 import java.io.StringReader;
  37 import java.util.*;
  38 import java.util.regex.Matcher;
  39 import java.util.regex.Pattern;
  40 
  41 /**
  42  * Support class to read and process a list of tests and test cases which are
  43  * known to fail during execution.  The intent is to allow better post-run
  44  * analysis of repetitive test runs, making is easier to find out what has
  45  * "changed" since the list was made.  This class is loosely based on the
  46  * exclude list, making it easy to interchange the files and tools.
  47  *
  48  * File format:
  49  * Test_URL[Test_Cases] BugID_List
  50  * The test URL rules are defined elsewhere, but it is critical that the test
  51  * names do not contain spaces and nothing before the BugID_List has any
  52  * whitespace.  The exact format of the BugID_List must simply conform to being
  53  * comma separated values, no whitespace or non-printable characters.
  54  * @since 4.4
  55  */
  56 public class KflSorter {
  57 
  58     /**
  59      *
  60      * @param kfl Effective known failures list to use - merge them in advance
  61      *    if multiple KFLs are needed.
  62      * @param trt Where to retrieve results from.
  63      * @param testcases True if test case analysis should be attempted.
  64      */
  65     KflSorter(KnownFailuresList kfl, TestResultTable trt, boolean testcases) {
  66         this.kfl = kfl;
  67         this.trt = trt;
  68         missing = new TreeSet<>();
  69         newFailures = new TreeSet<>();
  70         otherErrors = new TreeSet<>();
  71         fail2notrun = new TreeSet<>();
  72         fail2error = new TreeSet<>();
  73         fail2pass = new TreeSet<>();
  74         fail2fail = new TreeSet<>();
  75 
  76         tc_missing = new TreeSet<>();
  77         tc_fail2pass = new TreeSet<>();
  78         tc_fail2error = new TreeSet<>();
  79         tc_fail2notrun = new TreeSet<>();
  80         tc_newFailures = new TreeSet<>();
  81 
  82         enableTestCases = testcases;
  83     }
  84 
  85 //    KFL_Sorter(KnownFailuresList kfl, TestCaseList tcl) {
  86 //    }
  87     KnownFailuresList getKfl() {
  88         return kfl;
  89     }
  90 
  91     // NOTE - these are currently individual methods, but if each section
  92     //   became optional, a better API should be written to avoid massive
  93     //   number of set/get methods
  94     synchronized void setF2eEnabled(boolean state) {
  95         processF2e = state;
  96     }
  97 
  98     synchronized void setF2fEnabled(boolean state) {
  99         processF2f = state;
 100     }
 101 
 102     synchronized void setMissingEnabled(boolean state) {
 103         processMissing = state;
 104     }
 105 
 106     /**
 107      * Run the comparison of KFL vs set of results.
 108      * May take a long time to execute, performing the operation on a background
 109      * thread may be appropriate.
 110      * @param tests Set of results to compare to.
 111      * @return Number of comparison errors encountered.
 112      */
 113     synchronized int run(TestResultTable.TreeIterator iter) {
 114         TreeSet<TestResult>[] lists = new TreeSet[Status.NUM_STATES];
 115         int totalFound = 0;
 116 
 117         for (; iter.hasNext();) {
 118             TestResult tr = iter.next();
 119             Status s = tr.getStatus();
 120             TreeSet<TestResult> list = lists[s == null ? Status.NOT_RUN : s.getType()];
 121             list.add(tr);
 122             totalFound++;
 123         }
 124 
 125         return run(lists);
 126     }
 127 
 128     /**
 129      * Run the comparison of KFL vs set of results.
 130      * May take a long time to execute, performing the operation on a background
 131      * thread may be appropriate.
 132      * @param tests
 133      * @return Number of comparison problems encountered.
 134      */
 135     synchronized int run(TreeSet[] tests) {
 136         Iterator<KnownFailuresList.Entry> it = kfl.getIterator(false);
 137         int probs = 0;
 138         int tcprobs = 0;
 139 
 140         // iterator KFL entries
 141         for (;it.hasNext();) {
 142             KnownFailuresList.Entry entry = it.next();
 143             String url = entry.getRelativeURL();
 144             TestResult tr = trt.lookup(TestResult.getWorkRelativePath(url));
 145 
 146             if (tr == null) {
 147                 if (!processMissing)
 148                     continue;
 149 
 150                 if (missing.add(new TestDiff(url, null, Transitions.FAIL2MISSING)))
 151                     probs++;
 152 
 153                 if (enableTestCases) {
 154                     // add all test cases from this entry
 155                     tcprobs += addAllTestCases(entry, url, null, Transitions.TC_FAIL2MISSING,
 156                             tc_missing);
 157                 }
 158 
 159                 continue;
 160             }
 161 
 162             Map<String, Status> tcs = null;
 163             if (enableTestCases)
 164                 tcs = getTestCases(tr);
 165 
 166             if (tr.getStatus().isPassed()) {
 167                 // PASSED test on KFL
 168                 TestDiff diff = new TestDiff(url, tr, Transitions.FAIL2PASS);
 169                 if(fail2pass.add(diff))
 170                     probs++;
 171 
 172                 if (enableTestCases) {
 173                     //tcprobs += addAllTestCases(entry, url, tr, Transitions.TC_FAIL2PASS,
 174                     //        tc_fail2pass);
 175                     tcprobs += addStatusTestCases(url, tr,
 176                             Status.PASSED,
 177                             Transitions.TC_FAIL2PASS, tcs,
 178                             tc_fail2pass);
 179                 }
 180             }
 181             else if (tr.getStatus().isError()) {
 182                 // ERROR test on KFL
 183                 if (!processF2e)
 184                     continue;
 185 
 186                 TestDiff diff = new TestDiff(url, tr, Transitions.FAIL2ERROR);
 187                 if(fail2error.add(diff))
 188                     probs++;
 189 
 190                 if (enableTestCases) {
 191                     // add all test cases from this entry
 192                     tcprobs += addStatusTestCases(url, tr,
 193                             Status.ERROR,
 194                             Transitions.TC_FAIL2ERROR, tcs,
 195                             tc_fail2error);
 196                 }
 197             }
 198             else if (tr.getStatus().isNotRun()) {
 199                 // NOT RUN test on KFL
 200                 TestDiff diff = new TestDiff(url, tr, Transitions.FAIL2NOTRUN);
 201                 if(fail2notrun.add(diff))
 202                     probs++;
 203 
 204                 if (enableTestCases) {
 205                     // add all test cases from this entry
 206                     tcprobs += addStatusTestCases(url, tr,
 207                             Status.NOT_RUN,
 208                             Transitions.TC_FAIL2NOTRUN, tcs,
 209                             tc_fail2notrun);
 210                 }
 211             }
 212             else if (enableTestCases && tr.getStatus().isFailed()) {
 213                 // FAILED test on KFL
 214 
 215                 // look for new test cases which might be passing
 216                 // easy to do here because we are iterating the KFL
 217                 //tcprobs += findChangedCases(tr, tcs, entry);
 218             }
 219         }   // for
 220 
 221         // iterate failures, are they on the KFL?
 222         for(Object o: tests[Status.FAILED]) {
 223             TestResult tr = (TestResult)o;
 224             KnownFailuresList.Entry[] entries = kfl.find(tr.getTestName());
 225 
 226             if (entries == null || entries.length == 0) {
 227                 // not on KFL
 228                 TestDiff diff = new TestDiff(tr.getTestName(), tr,
 229                             Transitions.NEWFAILURES);
 230                 if (newFailures.add(diff))
 231                     probs++;
 232             }
 233             else {
 234                 if (!processF2f)
 235                     continue;
 236 
 237                 TestDiff diff = new TestDiff(tr.getTestName(), tr,
 238                             Transitions.FAIL2FAIL);
 239                 // does not count as a problem, so not added to probs
 240                 fail2fail.add(diff);
 241             }
 242 
 243             // assumes that failed test cases can only exist in failed tests
 244             if (enableTestCases) {
 245                 // tcs is likely looked up twice for failed tests,
 246                 // optimize to store & lookup
 247                 Map<String, Status> tcs = getTestCases(tr);
 248                 if (tcs != null && !tcs.isEmpty()) {
 249                     // only record test cases errors if test cases are listed
 250                     // in the KFL
 251 
 252                     KnownFailuresList.Entry[] full = kfl.find(tr.getTestName());
 253                     // different behavior if entire test listed vs
 254                     // specific test cases
 255                     boolean fullTestListed =  (full != null && !hasTestCases(full));
 256                     for (String name: tcs.keySet()) {
 257                         KnownFailuresList.Entry e = kfl.find(tr.getTestName(), name);
 258                         switch(tcs.get(name).getType()) {
 259                             case Status.FAILED:
 260                                 // check that it is listed
 261                                 if (!fullTestListed && e == null) {
 262                                     TestDiff td = new TestDiff(tr.getTestName(),
 263                                                 name, tr, Transitions.TC_NEWFAILURES);
 264                                     if(tc_newFailures.add(td))
 265                                         tcprobs++;
 266 
 267                                     // could optimize slightly to avoid repeated
 268                                     // adds if multiple test cases are new fails.
 269                                     // ensures that a tc-only differnece is also
 270                                     // reported in the non-tc new failures list
 271                                     TestDiff diff = new TestDiff(tr.getTestName(), tr,
 272                                             Transitions.NEWFAILURES);
 273                                     if (newFailures.add(diff)) {
 274                                         probs++;
 275                                     }
 276                                 }
 277                                 break;
 278                             case Status.PASSED:
 279                                 if((fullTestListed || e != null) &&
 280                                     tc_fail2pass.add(new TestDiff(tr.getTestName(),
 281                                             name, tr, Transitions.TC_FAIL2PASS)))
 282                                     tcprobs++;
 283                                 break;
 284                             case Status.ERROR:
 285 
 286                                 if((fullTestListed || e != null) &&
 287                                     tc_fail2error.add(new TestDiff(tr.getTestName(),
 288                                         name, tr, Transitions.TC_FAIL2ERROR)))
 289                                     tcprobs++;
 290                                 break;
 291                             case Status.NOT_RUN:
 292                                 if((fullTestListed || e != null) &&
 293                                     tc_fail2notrun.add(new TestDiff(tr.getTestName(),
 294                                         name, tr, Transitions.TC_FAIL2NOTRUN)))
 295                                     tcprobs++;
 296                                 break;
 297                             default:
 298                                 // oh well
 299                                 break;
 300                         }   // switch
 301                     }   // for
 302                 }
 303             }
 304         }   // for FAILED
 305 
 306                 // iterate errors, are they on the KFL?
 307         for(Object o: tests[Status.ERROR]) {
 308             TestResult tr = (TestResult)o;
 309             KnownFailuresList.Entry[] entries = kfl.find(tr.getTestName());
 310 
 311             if (entries == null || entries.length == 0) {
 312                 // not on KFL
 313                 // test is an error, but unrelated to the items listed on the KFL
 314                 TestDiff diff = new TestDiff(tr.getTestName(), tr,
 315                             Transitions.OTHER_ERRORS);
 316                 otherErrors.add(diff);
 317             }
 318 
 319             // no test case processing for this section
 320         }   // for ERROR
 321 
 322         errorCount = probs;
 323         tcErrorCount = tcprobs;
 324         return probs;
 325     }
 326 
 327     /**
 328      * Process a KFL entry looking for transitions from Failed to another
 329      * status.
 330      * Not sure this method implements an algorithm we want.
 331      * @deprecated Method not in use.
 332      */
 333     private int findChangedCases(final TestResult tr, final Map<String, Status> tcs,
 334             KnownFailuresList.Entry entry) {
 335         String kfltcl = entry.getTestCases();
 336 
 337         int problems = 0;
 338 
 339         if (kfltcl == null) {
 340             if (tcs != null && !tcs.isEmpty()) {
 341                 // entire test on KFL. check it for test cases which are not
 342                 // failed
 343                 for(String s: tcs.keySet()) {
 344                     Status stat = tcs.get(s);
 345 
 346                     if(stat.isError()) {
 347                         if (tc_fail2error.add(new TestDiff(
 348                             tr.getTestName(), s, tr, Transitions.TC_FAIL2ERROR)))
 349                             problems++;
 350                     }
 351                     else if (stat.isPassed()) {
 352                         if (tc_fail2pass.add(new TestDiff(
 353                             tr.getTestName(), s, tr, Transitions.TC_FAIL2PASS)))
 354                             problems++;
 355                     }
 356                     else if (stat.isNotRun()) {
 357                         if (tc_fail2notrun.add(new TestDiff(
 358                             tr.getTestName(), s, tr, Transitions.TC_FAIL2NOTRUN)))
 359                             problems++;
 360                     }
 361                 }   // for tcs
 362             }
 363         }
 364         else {
 365             // look for test cases listed on KFL, report mismatches (non failed ones)
 366             String[] kfltcs = StringArray.splitList(kfltcl, ",");
 367             for(String s: kfltcs) {
 368                 if (tcs != null && !tcs.isEmpty()) {
 369                     return addAllTestCases(entry, entry.getRelativeURL(), tr,
 370                             Transitions.TC_FAIL2MISSING, tc_missing);
 371                 }
 372 
 373                 Status stat = tcs.get(s);
 374 
 375                 if (stat == null) {
 376                     if (tc_missing.add(new TestDiff(
 377                         tr.getTestName(), s, tr, Transitions.TC_FAIL2MISSING)))
 378                         problems++;
 379                 }
 380                 else if(stat.isError()) {
 381                     if (tc_fail2error.add(new TestDiff(
 382                         tr.getTestName(), s, tr, Transitions.TC_FAIL2ERROR)))
 383                         problems++;
 384                 }
 385                 else if (stat.isPassed()) {
 386                     if (tc_fail2pass.add(new TestDiff(
 387                         tr.getTestName(), s, tr, Transitions.TC_FAIL2PASS)))
 388                         problems++;
 389                 }
 390                 else if (stat.isNotRun()) {
 391                     if (tc_fail2notrun.add(new TestDiff(
 392                         tr.getTestName(), s, tr, Transitions.TC_FAIL2NOTRUN)))
 393                         problems++;
 394                 }
 395             }   // for
 396         }
 397 
 398         return problems;
 399     }
 400 
 401     /**
 402      * Add all test cases from the KFL entry to the set.
 403      */
 404     private int addAllTestCases(final KnownFailuresList.Entry entry,
 405             final String url, final TestResult tr,
 406             Transitions t, Set<TestDiff> set) {
 407         // could check enableTestCases flag before processing
 408         // add all test cases from this entry
 409         int problems = 0;
 410 
 411         String[] tcs = entry.getTestCaseList();
 412         if (tcs == null || tcs.length == 0)
 413             return 0;
 414 
 415         for (String s: tcs) {
 416             if (set.add(new TestDiff(url, s, tr, t)))
 417                 problems++;
 418         }
 419 
 420         return problems;
 421     }
 422 
 423     private int addListedTestCases(final KnownFailuresList.Entry entry,
 424             final String url, final TestResult tr,
 425             Transitions t, TreeSet set) {
 426         int problems = 0;
 427 
 428         String[] tcs = entry.getTestCaseList();
 429         if (tcs == null || tcs.length == 0)
 430             return 0;
 431 
 432         for (String s: tcs) {
 433 
 434         }
 435 
 436         return problems;
 437     }
 438 
 439     /**
 440      * Add all test cases for a test which match the given status to the tree set.
 441      * @param entry The corresponding KFL entry.
 442      * @param
 443      * @param set the data set to add the selected test cases to
 444      * @return the number of test cases which matched the status
 445      */
 446 //    private int addStatusTestCases(final KnownFailuresList.Entry entry,
 447 //            final String url, final TestResult tr, int status,
 448 //            Transitions t, Map<String, Status> tcs, TreeSet set) {
 449 //        // could check enableTestCases flag before processing
 450 //
 451 //        // add all test cases from this entry
 452 //        int problems = 0;
 453 //
 454 //        //String[] tcs = entry.getTestCaseList();
 455 //        if (tcs == null || tcs.isEmpty())
 456 //            return 0;
 457 //
 458 //        for (String key: tcs.keySet()) {
 459 //            Status stat = tcs.get(key);
 460 //
 461 //            if (stat.getType() == status) {
 462 //                if(set.add(new TestDiff(url, key, tr, t)))
 463 //                    problems++;
 464 //            }
 465 //        }
 466 //
 467 //        return problems;
 468 //    }
 469 
 470 
 471     /**
 472      * Add all test cases from the test result, listed in the KFL entry
 473      * which match the given status.
 474      * Effectively - iterate test cases in result (tcs), if it is listed in the
 475      * KFL and matches the given status, add to the given set.
 476      * @param entry The KFL entry.
 477      * @param tcs Test case results for the given test.
 478      * @param status the status type to match
 479      * @param set the data set to add the selected test cases to
 480      * @return the number of test cases which were added to the set
 481      * @see com.sun.javatest.Status
 482      */
 483     private int addStatusTestCases(
 484             final String url, final TestResult tr, int status,
 485             Transitions t, Map<String, Status> trtcs, Set<TestDiff> set) {
 486         // could check enableTestCases flag before processing
 487         int problems = 0;
 488 
 489         if (trtcs == null || trtcs.isEmpty()) {
 490             return 0;
 491         }
 492         else {
 493             for (String key: trtcs.keySet()) {
 494                 Status stat = trtcs.get(key);
 495                 KnownFailuresList.Entry e = kfl.find(url, key);
 496                 KnownFailuresList.Entry[] full = kfl.find(tr.getTestName());
 497                 // different behavior if entire test listed vs
 498                 // specific test cases
 499                 boolean fullTestListed = (full != null && !hasTestCases(full));
 500 
 501                 // test case not listed in KFL
 502                 // no need to list a passing test case which wasn't
 503                 // listed as a failure in the KFL
 504                 // could be a new test case, etc.
 505                 if (!fullTestListed && e == null)
 506                     continue;
 507 
 508                 if (stat.getType() == status) {
 509                     TestDiff diff = new TestDiff(url, key, tr, t);
 510                     if (fullTestListed && full != null && full.length > 0)
 511                         diff.setKflEntry(full[0]);
 512 
 513                     if(set.add(diff))
 514                         problems++;
 515                 }
 516             }   // for
 517         }
 518 
 519         return problems;
 520     }
 521 
 522 
 523     private boolean hasTestCases(final KnownFailuresList.Entry[] es) {
 524         if (es == null || es.length == 0)
 525             return false;
 526 
 527         for(KnownFailuresList.Entry e: es) {
 528             if(e.getTestCases() != null)
 529                 return true;
 530         }
 531 
 532         return false;
 533     }
 534 
 535     synchronized SortedSet<TestDiff> getSet(Transitions id) {
 536         switch (id) {
 537             case FAIL2PASS: return fail2pass;
 538             case FAIL2MISSING: return missing;
 539             case NEWFAILURES: return newFailures;
 540             case OTHER_ERRORS: return otherErrors;
 541             case FAIL2NOTRUN: return fail2notrun;
 542             case FAIL2ERROR: return fail2error;
 543             case FAIL2FAIL: return fail2fail;
 544 
 545             case TC_FAIL2PASS: return tc_fail2pass;
 546             case TC_FAIL2MISSING: return tc_missing;
 547             case TC_NEWFAILURES: return tc_newFailures;
 548             case TC_FAIL2NOTRUN: return tc_fail2notrun;
 549             case TC_FAIL2ERROR: return tc_fail2error;
 550             default: return null;
 551         } // switch
 552     }
 553 
 554     synchronized int getErrorCount() {
 555         return errorCount;
 556     }
 557 
 558     synchronized int getTestCasesErrorCount() {
 559         return tcErrorCount;
 560     }
 561 
 562 
 563     private static Map<String, Status> getTestCases(final TestResult tr) {
 564         Map<String, Status> result = new LinkedHashMap<>();
 565 
 566         if (tr.isShrunk() && tr.isReloadable())
 567             tr.getSectionTitles();
 568 
 569         int sCount = tr.getSectionCount();
 570         if (sCount == 0 && tr.getStatus().getType() != Status.NOT_RUN) {
 571             //sCount = tr.getSectionCount();
 572         }
 573 
 574         for (int i = 0; i < sCount; i++) {
 575             try {
 576                 String sectionOut = tr.getSection(i).getOutput("out1");
 577                 if (sectionOut == null) {
 578                     continue;
 579                 }
 580                 BufferedReader reader = new BufferedReader(new StringReader(
 581                         sectionOut));
 582                 String s = reader.readLine();
 583                 while (s != null) {
 584                     Matcher m = testCasePattern.matcher(s);
 585                     if (m.matches()) {
 586                         String tcName = m.group(1);
 587                         // checking for space helps eliminate false matches
 588                         if (tcName == null || tcName.contains(" ") ||
 589                             tcName.contains("\t")) {
 590                             s = reader.readLine();
 591                             continue;
 592                         }
 593                         Status stat = Status.parse(m.group(2));
 594                         if (result.containsKey(tcName)){
 595                             // testcases with the same name are marked as passed
 596                             // only if all these testcases are passed
 597                             if (!stat.isPassed()){
 598                                 result.put(tcName, stat);
 599                             }
 600                         }
 601                         else {
 602                             result.put(tcName, stat);
 603                         }
 604                     }
 605                     s = reader.readLine();
 606                 }
 607             } catch (IOException e) {
 608                 // TODO Auto-generated catch block
 609             } catch (TestResult.ReloadFault e) {
 610                 // TODO Auto-generated catch block
 611             }
 612         }
 613 
 614         return (result.isEmpty() ? null : result);
 615     }
 616 
 617     /**
 618      * Created for each result which somehow does not match what was expected
 619      * based on the KFL. Using this class allows the analysis to be done once
 620      * then queried again and again for different purposes.
 621      */
 622     public static class TestDiff implements Comparable {
 623         public TestDiff(String url, TestResult tr, Transitions type) {
 624             this.tr = tr;
 625             this.url = url;
 626         }
 627 
 628         public TestDiff(String url, String tc, TestResult tr, Transitions type) {
 629             this(url, tr, type);
 630             this.tc = tc;
 631             // resultMismatch =
 632             // caseMismatch =
 633         }
 634 
 635         public TestResult getTestResult() {
 636             return tr;
 637         }
 638 
 639         /**
 640          * Is the mismatch concerning the test's main result?
 641          * @return True if the result status is not the same as the result expected
 642          *    based on the KFL.  False if the main result matches.
 643          */
 644         public boolean isTestMismatch() {
 645             return resultMismatch;
 646         }
 647 
 648 
 649         /**
 650          * Get the full name for this entry, including the test case.
 651          * Most useful for display purposes.
 652          * @return An easily human readable string.
 653          * @see #getTestName()
 654          * @see #getTestCase()
 655          */
 656         public String getName() {
 657             String u = (url != null ? url : tr.getTestName());
 658             if (tc == null) {
 659                 return u;
 660             }
 661             else {
 662                 return u + "[" + tc + "]";
 663             }
 664         }
 665 
 666         /**
 667          * Get the name of the test involved in this diff, not including the
 668          * test case portion if that applies.
 669          * @see #getName()
 670          */
 671         public String getTestName() {
 672             return url;
 673         }
 674 
 675         /**
 676          * Get the list of the test case(s).
 677          * @return Null if there are no test cases associated, otherwise a
 678          *    comma separated string of test case names.
 679          */
 680         public String getTestCase() {
 681             return tc;
 682         }
 683 
 684         /**
 685          * Not normally used, but can be used as a secondary way to get
 686          * the associated KFL entry.  Typically this value will be null.
 687          * @return A KFL entry which caused this diff.
 688          */
 689         public KnownFailuresList.Entry getKflEntry() {
 690             return kflEntry;
 691         }
 692 
 693         /**
 694          * Not normally used, but can be used as a backup if there is a
 695          * special case where looking up the entry later would fail.
 696          * @param e The KFL entry to associate with this diff.
 697          */
 698         public void setKflEntry(KnownFailuresList.Entry e) {
 699             kflEntry = e;
 700         }
 701 
 702         @Override
 703         public int compareTo(Object o) {
 704             TestDiff e = (TestDiff) o;
 705             int n = getName().compareTo(e.getName());
 706 /*          if (n == 0) {
 707                 if (testCase == null && e.testCase == null)
 708                     return 0;
 709                 else if (testCase == null)
 710                     return -1;
 711                 else if (e.testCase == null)
 712                     return +1;
 713                 else
 714                     return testCase.compareTo(e.testCase);
 715             }
 716             else
 717                 return n;*/
 718             return n;
 719         }
 720 
 721         private TestResult tr;
 722         private String url;
 723         private String tc;
 724         private boolean resultMismatch = false;
 725         private boolean caseMismatch = false;
 726         private KnownFailuresList.Entry kflEntry;
 727     }
 728 
 729     public enum Transitions { FAIL2PASS, FAIL2ERROR, FAIL2MISSING, FAIL2NOTRUN, FAIL2FAIL, NEWFAILURES,
 730         OTHER_ERRORS, TC_FAIL2MISSING, TC_FAIL2PASS, TC_PASS2ERROR, TC_FAIL2NOTRUN, TC_FAIL2ERROR, TC_NEWFAILURES }
 731 //    interface KflObserver {
 732 //        public void passToFail(TestResult tr);
 733 //        public void failToPass(TestResult tr);
 734 //    }
 735     protected SortedSet<TestDiff> fail2pass;
 736     protected SortedSet<TestDiff> fail2error;
 737     protected SortedSet<TestDiff> fail2notrun;
 738     protected SortedSet<TestDiff> missing;
 739     protected SortedSet<TestDiff> newFailures;
 740     protected SortedSet<TestDiff> otherErrors;
 741     protected SortedSet<TestDiff> fail2fail;
 742 
 743     protected SortedSet<TestDiff> tc_missing;
 744     protected SortedSet<TestDiff> tc_fail2pass;
 745     protected SortedSet<TestDiff> tc_fail2error;
 746     protected SortedSet<TestDiff> tc_fail2notrun;
 747     protected SortedSet<TestDiff> tc_newFailures;
 748 
 749     protected KnownFailuresList kfl;
 750     //protected TestCaseList tcl;
 751     protected TestResultTable trt;
 752     protected int errorCount;
 753     protected int tcErrorCount;
 754     private boolean enableTestCases;
 755     private boolean processF2f, processF2e, processMissing = true;
 756 
 757     protected static final Pattern testCasePattern = Pattern //.compile("^(\\S+): (Passed\\.|Failed\\.|Error\\.|Not\\ run\\.)(.*)");
 758             .compile("^(.*): (Passed\\.|Failed\\.|Error\\.|Not\\ run\\.)(.*)");
 759 }