1   /***
2    * <copyright>
3    *  Copyright 1997-2002 BBNT Solutions, LLC
4    *  under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5    *
6    *  This program is free software; you can redistribute it and/or modify
7    *  it under the terms of the Cougaar Open Source License as published by
8    *  DARPA on the Cougaar Open Source Website (www.cougaar.org).
9    *
10   *  THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11   *  PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12   *  IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13   *  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14   *  ANY WARRANTIES AS TO NON-INFRINGEMENT.  IN NO EVENT SHALL COPYRIGHT
15   *  HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16   *  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17   *  TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18   *  PERFORMANCE OF THE COUGAAR SOFTWARE.
19   * </copyright>
20   *
21   * Created on Aug 26, 2002
22   */
23  package test.net.sourceforge.pmd.stat;
24  
25  import junit.framework.AssertionFailedError;
26  import junit.framework.TestCase;
27  import net.sourceforge.pmd.Report;
28  import net.sourceforge.pmd.Rule;
29  import net.sourceforge.pmd.RuleContext;
30  import net.sourceforge.pmd.stat.DataPoint;
31  import net.sourceforge.pmd.stat.Metric;
32  import net.sourceforge.pmd.stat.StatisticalRule;
33  
34  import java.util.ArrayList;
35  import java.util.Collections;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.Random;
39  
40  /***
41   * This class tests the Statistical Rules in PMD.
42   *
43   * The idea is, that we fill up 999 datapoints into
44   * the Stat Rule, and then throw random parameters
45   * at it.
46   *
47   * The three parameters which are checked are:
48   * 		sigma - # Sigmas over the mean.
49   * 		topscore - Only the top 5 or so items.
50   * 		minimum - Only things of score 10 or better
51   *
52   * When more than one parameter is lumped together, then
53   * we expect the one which would return the fewest to
54   * determine what gets sent back.
55   *
56   * So, we throw each collection of parameters, where each
57   * one is a different order into the system.  We check the
58   * results off of what the smallest value should be.
59   *
60   * If you are going to work with StatisticalRule any, please
61   * bump the "NUM_TESTS" number up to something like 128.  That
62   * way you are more likely to identify problems.  It is set low
63   * now to make building and running tests easier (when we aren't
64   * touching the file.)
65   *
66   * Note also, that when verifying the Sigma, I wasn't quite able
67   * to determine how many results it would return (it would vary
68   * from -2 to 2 of what I expected.)  That is what the delta
69   * parameter on the verify method takes.  If you can figure it
70   * out exactly, (without stealing code from the StatRule) then
71   * feel free to change it and tighten the deltas.
72   */
73  public class StatisticalRuleTest extends TestCase {
74  
75      private static final int POINTS = 100;
76  
77      private DataPoint points[] = new DataPoint[POINTS];
78      private MockStatisticalRule IUT = null;
79      private String testName = null;
80      private Random random = new Random();
81  
82      public static final double MAX_MINIMUM = POINTS;
83      public static final double NO_MINIMUM = -1.0;
84      public static final double MAX_SIGMA = 5.0;
85      public static final double NO_SIGMA = -1.0;
86      public static final int MIN_TOPSCORE = 0;
87      public static final int NO_TOPSCORE = -1;
88  
89  
90      public static final double MEAN = 49.5;
91      public static final double SIGMA = 29.0115;
92      public static final int NUM_TESTS = 1;
93  
94      public static final double DELTA = 0.005;
95  
96      public StatisticalRuleTest(String name) {
97          super(name);
98          this.testName = name;
99      }
100 
101     public void setUp() {
102         IUT = new MockStatisticalRule();
103         if (testName.endsWith("0")) {
104             for (int i = 0; i < POINTS; i++) {
105                 points[i] = new DataPoint();
106                 points[i].setScore(1.0 * i);
107                 points[i].setLineNumber(i);
108                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
109 
110                 IUT.addDataPoint(points[i]);
111             }
112         } else if (testName.endsWith("1")) {
113             for (int i = POINTS-1; i >= 0; i--) {
114                 points[i] = new DataPoint();
115                 points[i].setScore(1.0 * i);
116                 points[i].setLineNumber(i);
117                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
118 
119                 IUT.addDataPoint(points[i]);
120             }
121         } else {
122             List lPoints = new ArrayList();
123             for (int i = 0; i < POINTS; i++) {
124                 DataPoint point = new DataPoint();
125                 point.setScore(1.0 * i);
126                 point.setLineNumber(i);
127                 point.setMessage("DataPoint[" + Integer.toString(i) + "]");
128 
129                 lPoints.add(point);
130             }
131 
132             Collections.shuffle(lPoints);
133             for (int i = 0; i < POINTS; i++) {
134                 IUT.addDataPoint((DataPoint) lPoints.get(i));
135             }
136         }
137 
138     }
139 
140     /***
141      * This test verifies that the Stat rule creates a Metric,
142      * with the proper values.
143      */
144     public void testMetrics() throws Throwable {
145         Report report = makeReport(IUT);
146         Iterator metrics = report.metrics();
147 
148         assertTrue(metrics.hasNext());
149         Object o = metrics.next();
150 
151         assertTrue(o instanceof Metric);
152         Metric m = (Metric) o;
153 
154         assertEquals("test.net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
155 
156         assertEquals(0.0, m.getLowValue(), 0.05);
157         assertEquals(POINTS -1.0, m.getHighValue(), 0.05);
158         assertEquals(MEAN, m.getAverage(), 0.05);
159         assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
160     }
161 
162     /***
163      * This returns a Random value for Sigma which will
164      * return some values.
165      */
166     public double randomSigma() {
167         return random.nextDouble() * 1.0;
168     }
169 
170     /***
171      * This returns a Random value for Sigma which value
172      * is greater than the parameter.
173      */
174     public double randomSigma(int minimum) {
175         double minSigma = ((POINTS -1 - minimum) - MEAN) / SIGMA;
176 
177         if ((minSigma <= 0) || (minSigma > 2))
178             return randomSigma();
179 
180         return minSigma + (random.nextDouble() * (2 - minSigma));
181     }
182 
183     /***
184      * This returns the expected number of results when
185      * the Sigma rating is the smallest.
186      */
187     public int expectedSigma(double sigma) {
188         long expectedMin = Math.round(MEAN + (sigma * SIGMA));
189 
190         if (((POINTS -1) - expectedMin) < 0)
191             return 0;
192         return (POINTS -1) - (int) expectedMin;
193     }
194 
195     /***
196      * This generates a random minimum value for testing.
197      */
198     public double randomMinimum() {
199         return random.nextDouble() * (POINTS -1);
200     }
201 
202     /***
203      * This generates a random minimum value for which fewer
204      * results would be returned.
205      */
206     public double randomMinimum(int minimum) {
207         double diffTarget = 1.0 * (POINTS -1 - minimum);
208         return (random.nextDouble() * minimum) + diffTarget;
209     }
210 
211     /***
212      * This returns the expected number of reports.
213      *
214      * If the Minimum comes in at 521.569 then we expect
215      * 522, 523, ... 999 will pass.
216      */
217     public int expectedMinimum(double minimum) {
218         Double d = new Double(minimum);
219         return POINTS -1 - d.intValue();
220     }
221 
222     public void testExpectedMinimum() {
223         for (int i = 0; i < POINTS -1; i++) {
224             assertEquals("Integer Min", POINTS -1 - i, expectedMinimum(i * 1.0));
225             assertEquals("Double Min", POINTS -1 - i, expectedMinimum((i * 1.0) + 0.5));
226         }
227     }
228 
229     /***
230      * This returns a random value for Top Score.
231      */
232     public int randomTopScore() {
233         return random.nextInt(POINTS -1);
234     }
235 
236     /***
237      * This will return a random value for the Top Score
238      * which will return more than the minimum provided.
239      */
240     public int randomTopScore(double target) {
241         if (target < 0)
242             return 0;
243 
244         return random.nextInt((new Double(target)).intValue());
245     }
246 
247     /***
248      * This will return the expected number of results
249      * with the given Top Score.
250      */
251     public int expectedTopScore(int target) {
252         return target;
253     }
254 
255     // Test Single Datapoint
256     public void testSingleDatapoint() {
257         StatisticalRule IUT = new MockStatisticalRule();
258 
259         DataPoint point = new DataPoint();
260         point.setScore(POINTS + 1.0);
261         point.setLineNumber(POINTS + 1);
262         point.setMessage("SingleDataPoint");
263 
264         IUT.addProperty("minimum", Integer.toString(POINTS));
265 
266         IUT.addDataPoint(point);
267 
268         Report report = makeReport(IUT);
269 
270         assertEquals("Expecting only one result.", 1, report.size());
271     }
272 
273     // Okay, we have three properties we need to
274     // test in Combination:
275     //  S = Sigma
276     //  T = Top Score
277     //  M = Minimum
278     //
279     // They are listed in decreasing order of what
280     // to expect.
281     //
282     // Thus testSM() should have the Sigma less than
283     // the minimum, so we expect the Minimum # of results.
284     //
285 
286     public void testS() throws Throwable {
287         verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
288 
289         for (int i = 0; i < NUM_TESTS; i++) {
290             double sigma = randomSigma();
291             verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
292         }
293     }
294 
295     public void testS1() throws Throwable {
296         testS();
297     }
298 
299     public void testS2() throws Throwable {
300         testS();
301     }
302 
303     public void testS3() throws Throwable {
304         testS();
305     }
306 
307     public void testS4() throws Throwable {
308         testS();
309     }
310 
311     public void testS5() throws Throwable {
312         testS();
313     }
314 
315 
316     public void testT() throws Throwable {
317         verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
318 
319         for (int i = 0; i < NUM_TESTS; i++) {
320             int topScore = randomTopScore();
321             verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
322         }
323     }
324 
325     public void testT1() throws Throwable {
326         testT();
327     }
328 
329     public void testT2() throws Throwable {
330         testT();
331     }
332 
333     public void testT3() throws Throwable {
334         testT();
335     }
336 
337     public void testT4() throws Throwable {
338         testT();
339     }
340 
341     public void testT5() throws Throwable {
342         testT();
343     }
344 
345     public void testM() throws Throwable {
346         verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
347 
348         for (int i = 0; i < NUM_TESTS; i++) {
349             double minimum = randomMinimum();
350             verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
351         }
352     }
353 
354     public void testM1() throws Throwable {
355         testM();
356     }
357 
358     public void testM2() throws Throwable {
359         testM();
360     }
361 
362     public void testM3() throws Throwable {
363         testM();
364     }
365 
366     public void testM4() throws Throwable {
367         testM();
368     }
369 
370     public void testM5() throws Throwable {
371         testM();
372     }
373 
374     public void testST() throws Throwable {
375         verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
376 
377         for (int i = 0; i < NUM_TESTS; i++) {
378             double sigma = randomSigma();
379             int topScore = randomTopScore(expectedSigma(sigma));
380 
381             verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
382         }
383     }
384 
385     public void testST1() throws Throwable {
386         testST();
387     }
388 
389     public void testST2() throws Throwable {
390         testST();
391     }
392 
393     public void testST3() throws Throwable {
394         testST();
395     }
396 
397     public void testST4() throws Throwable {
398         testST();
399     }
400 
401     public void testST5() throws Throwable {
402         testST();
403     }
404 
405     public void testTS() throws Throwable {
406         verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
407 
408         for (int i = 0; i < NUM_TESTS; i++) {
409             int topScore = randomTopScore();
410             double sigma = randomSigma(expectedTopScore(topScore));
411 
412             verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
413         }
414     }
415 
416     public void testTS1() throws Throwable {
417         testTS();
418     }
419 
420     public void testTS2() throws Throwable {
421         testTS();
422     }
423 
424     public void testTS3() throws Throwable {
425         testTS();
426     }
427 
428     public void testTS4() throws Throwable {
429         testTS();
430     }
431 
432     public void testTS5() throws Throwable {
433         testTS();
434     }
435 
436     public void testSM() throws Throwable {
437         verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
438         for (int i = 0; i < NUM_TESTS; i++) {
439             double sigma = randomSigma();
440             double minimum = randomMinimum(expectedSigma(sigma));
441 
442             verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
443         }
444 
445     }
446 
447     public void testSM1() throws Throwable {
448         testSM();
449     }
450 
451     public void testSM2() throws Throwable {
452         testSM();
453     }
454 
455     public void testSM3() throws Throwable {
456         testSM();
457     }
458 
459     public void testSM4() throws Throwable {
460         testSM();
461     }
462 
463     public void testSM5() throws Throwable {
464         testSM();
465     }
466 
467 
468     public void testMS() throws Throwable {
469         verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
470         for (int i = 0; i < NUM_TESTS; i++) {
471             double minimum = randomMinimum();
472             double sigma = randomSigma(expectedMinimum(minimum));
473 
474             verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
475         }
476     }
477 
478     public void testMS1() throws Throwable {
479         testMS();
480     }
481 
482     public void testMS2() throws Throwable {
483         testMS();
484     }
485 
486     public void testMS3() throws Throwable {
487         testMS();
488     }
489 
490     public void testMS4() throws Throwable {
491         testMS();
492     }
493 
494     public void testMS5() throws Throwable {
495         testMS();
496     }
497 
498 
499     public void testTM() throws Throwable {
500         verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
501         for (int i = 0; i < NUM_TESTS; i++) {
502             int topScore = randomTopScore();
503             double minimum = randomMinimum(expectedTopScore(topScore));
504 
505             verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
506         }
507     }
508 
509     public void testTM1() throws Throwable {
510         testTM();
511     }
512 
513     public void testTM2() throws Throwable {
514         testTM();
515     }
516 
517     public void testTM3() throws Throwable {
518         testTM();
519     }
520 
521     public void testTM4() throws Throwable {
522         testTM();
523     }
524 
525     public void testTM5() throws Throwable {
526         testTM();
527     }
528 
529 
530     public void testMT() throws Throwable {
531         verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
532         for (int i = 0; i < NUM_TESTS; i++) {
533             double minimum = randomMinimum();
534             int topScore = randomTopScore(expectedMinimum(minimum));
535 
536             verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
537         }
538     }
539 
540     public void testMT1() throws Throwable {
541         testMT();
542     }
543 
544     public void testMT2() throws Throwable {
545         testMT();
546     }
547 
548     public void testMT3() throws Throwable {
549         testMT();
550     }
551 
552     public void testMT4() throws Throwable {
553         testMT();
554     }
555 
556     public void testMT5() throws Throwable {
557         testMT();
558     }
559 
560 
561     public void testSTM() throws Throwable {
562         double sigma = randomSigma();
563         verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
564 
565         for (int i = 0; i < NUM_TESTS; i++) {
566             sigma = randomSigma();
567             int topScore = randomTopScore(expectedSigma(sigma));
568             double minimum = randomMinimum(expectedTopScore(topScore));
569 
570             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
571         }
572     }
573 
574     public void testSTM1() throws Throwable {
575         testSTM();
576     }
577 
578     public void testSTM2() throws Throwable {
579         testSTM();
580     }
581 
582     public void testSTM3() throws Throwable {
583         testSTM();
584     }
585 
586     public void testSTM4() throws Throwable {
587         testSTM();
588     }
589 
590     public void testSTM5() throws Throwable {
591         testSTM();
592     }
593 
594     public void testSMT() throws Throwable {
595         double sigma = randomSigma();
596         verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
597 
598         for (int i = 0; i < NUM_TESTS; i++) {
599             sigma = randomSigma();
600             double minimum = randomMinimum(expectedSigma(sigma));
601             int topScore = randomTopScore(expectedMinimum(minimum));
602 
603             verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
604         }
605     }
606 
607     public void testSMT1() throws Throwable {
608         testSMT();
609     }
610 
611     public void testSMT2() throws Throwable {
612         testSMT();
613     }
614 
615     public void testSMT3() throws Throwable {
616         testSMT();
617     }
618 
619     public void testSMT4() throws Throwable {
620         testSMT();
621     }
622 
623     public void testSMT5() throws Throwable {
624         testSMT();
625     }
626 
627     public void testTSM() throws Throwable {
628         int topScore = randomTopScore();
629         verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
630 
631         for (int i = 0; i < NUM_TESTS; i++) {
632             topScore = randomTopScore();
633             double sigma = randomSigma(expectedTopScore(topScore));
634             double minimum = randomMinimum(expectedSigma(sigma));
635 
636             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
637         }
638     }
639 
640     public void testTSM1() throws Throwable {
641         testTSM();
642     }
643 
644     public void testTSM2() throws Throwable {
645         testTSM();
646     }
647 
648     public void testTSM3() throws Throwable {
649         testTSM();
650     }
651 
652     public void testTSM4() throws Throwable {
653         testTSM();
654     }
655 
656     public void testTSM5() throws Throwable {
657         testTSM();
658     }
659 
660     public void testTMS() throws Throwable {
661         int topScore = randomTopScore();
662         verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
663 
664         for (int i = 0; i < NUM_TESTS; i++) {
665             topScore = randomTopScore();
666             double minimum = randomMinimum(expectedTopScore(topScore));
667             double sigma = randomSigma(expectedMinimum(minimum));
668 
669             verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
670         }
671     }
672 
673     public void testTMS1() throws Throwable {
674         testTMS();
675     }
676 
677     public void testTMS2() throws Throwable {
678         testTMS();
679     }
680 
681     public void testTMS3() throws Throwable {
682         testTMS();
683     }
684 
685     public void testTMS4() throws Throwable {
686         testTMS();
687     }
688 
689     public void testTMS5() throws Throwable {
690         testTMS();
691     }
692 
693     /***
694      * Verifies what happens when you pass these parameters
695      * into the thing.  DELTA is the amount of error allowed.
696      * Usually DELTA is only used for Sigma, as we really can't
697      * calculate it exactly.
698      */
699 
700     public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
701         try {
702             setUp();
703             if (sigma >= 0) {
704                 IUT.addProperty("sigma", Double.toString(sigma));
705             }
706 
707             if (minimum >= 0) {
708                 IUT.addProperty("minimum", Double.toString(minimum));
709             }
710 
711             if (topScore >= 0) {
712                 IUT.addProperty("topscore", Integer.toString(topScore));
713             }
714 
715             Report report = makeReport(IUT);
716             if (delta == 0) {
717                 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
718             } else {
719                 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
720 
721                 assertTrue(assertStr, report.size() >= (expected - delta));
722                 assertTrue(assertStr, report.size() <= (expected + delta));
723             }
724         } catch (AssertionFailedError afe) {
725             System.err.println("******** " + testName + " ***********");
726             if (sigma != NO_SIGMA) {
727                 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
728             }
729 
730             if (minimum != NO_MINIMUM) {
731                 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
732             }
733 
734             if (topScore != NO_TOPSCORE) {
735                 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
736             }
737 
738             throw afe;
739 
740         }
741     }
742 
743     public Report makeReport(Rule IUT) {
744         List list = new ArrayList();
745         Report report = new Report();
746 
747         RuleContext ctx = new RuleContext();
748         ctx.setReport(report);
749         ctx.setSourceCodeFilename(testName);
750 
751         IUT.apply(list, ctx);
752 
753         return report;
754     }
755 }