1   /***
2    * <copyright>
3    *  Copyright 1997-2002 BBNT Solutions, LLC
4    *  under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5    *
6    *  This program is free software; you can redistribute it and/or modify
7    *  it under the terms of the Cougaar Open Source License as published by
8    *  DARPA on the Cougaar Open Source Website (www.cougaar.org).
9    *
10   *  THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11   *  PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12   *  IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13   *  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14   *  ANY WARRANTIES AS TO NON-INFRINGEMENT.  IN NO EVENT SHALL COPYRIGHT
15   *  HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16   *  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17   *  TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18   *  PERFORMANCE OF THE COUGAAR SOFTWARE.
19   * </copyright>
20   *
21   * Created on Aug 26, 2002
22   */
23  package test.net.sourceforge.pmd.stat;
24  
25  import junit.framework.AssertionFailedError;
26  import junit.framework.TestCase;
27  import net.sourceforge.pmd.Report;
28  import net.sourceforge.pmd.Rule;
29  import net.sourceforge.pmd.RuleContext;
30  import net.sourceforge.pmd.ast.SimpleJavaNode;
31  import net.sourceforge.pmd.ast.SimpleNode;
32  import net.sourceforge.pmd.stat.DataPoint;
33  import net.sourceforge.pmd.stat.Metric;
34  import net.sourceforge.pmd.stat.StatisticalRule;
35  import net.sourceforge.pmd.symboltable.SourceFileScope;
36  
37  import java.util.ArrayList;
38  import java.util.Collections;
39  import java.util.Iterator;
40  import java.util.List;
41  import java.util.Random;
42  
43  /***
44   * This class tests the Statistical Rules in PMD.
45   * <p/>
46   * The idea is, that we fill up 999 datapoints into
47   * the Stat Rule, and then throw random parameters
48   * at it.
49   * <p/>
50   * The three parameters which are checked are:
51   * sigma - # Sigmas over the mean.
52   * topscore - Only the top 5 or so items.
53   * minimum - Only things of score 10 or better
54   * <p/>
55   * When more than one parameter is lumped together, then
56   * we expect the one which would return the fewest to
57   * determine what gets sent back.
58   * <p/>
59   * So, we throw each collection of parameters, where each
60   * one is a different order into the system.  We check the
61   * results off of what the smallest value should be.
62   * <p/>
63   * If you are going to work with StatisticalRule any, please
64   * bump the "NUM_TESTS" number up to something like 128.  That
65   * way you are more likely to identify problems.  It is set low
66   * now to make building and running tests easier (when we aren't
67   * touching the file.)
68   * <p/>
69   * Note also, that when verifying the Sigma, I wasn't quite able
70   * to determine how many results it would return (it would vary
71   * from -2 to 2 of what I expected.)  That is what the delta
72   * parameter on the verify method takes.  If you can figure it
73   * out exactly, (without stealing code from the StatRule) then
74   * feel free to change it and tighten the deltas.
75   */
76  public class StatisticalRuleTest extends TestCase {
77  
78      private static final int POINTS = 100;
79  
80      private DataPoint points[] = new DataPoint[POINTS];
81      private MockStatisticalRule IUT = null;
82      private String testName = null;
83      private Random random = new Random();
84  
85      public static final double MAX_MINIMUM = POINTS;
86      public static final double NO_MINIMUM = -1.0;
87      public static final double MAX_SIGMA = 5.0;
88      public static final double NO_SIGMA = -1.0;
89      public static final int MIN_TOPSCORE = 0;
90      public static final int NO_TOPSCORE = -1;
91  
92  
93      public static final double MEAN = 49.5;
94      public static final double SIGMA = 29.0115;
95      public static final int NUM_TESTS = 1;
96  
97      public static final double DELTA = 0.005;
98  
99      public StatisticalRuleTest(String name) {
100         super(name);
101         this.testName = name;
102     }
103 
104     public void setUp() {
105         IUT = new MockStatisticalRule();
106         if (testName.endsWith("0")) {
107             for (int i = 0; i < POINTS; i++) {
108                 points[i] = new DataPoint();
109                 points[i].setScore(1.0 * i);
110                 SimpleNode s = new SimpleJavaNode(1);
111                 s.setScope(new SourceFileScope("foo"));
112                 s.testingOnly__setBeginLine(i);
113                 s.testingOnly__setBeginColumn(1);
114                 points[i].setNode(s);
115                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
116 
117                 IUT.addDataPoint(points[i]);
118             }
119         } else if (testName.endsWith("1")) {
120             for (int i = POINTS - 1; i >= 0; i--) {
121                 points[i] = new DataPoint();
122                 points[i].setScore(1.0 * i);
123                 SimpleNode s = new SimpleJavaNode(1);
124                 s.setScope(new SourceFileScope("foo"));
125                 s.testingOnly__setBeginLine(i);
126                 s.testingOnly__setBeginColumn(1);
127                 points[i].setNode(s);
128                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
129 
130                 IUT.addDataPoint(points[i]);
131             }
132         } else {
133             List lPoints = new ArrayList();
134             for (int i = 0; i < POINTS; i++) {
135                 points[i] = new DataPoint();
136                 points[i].setScore(1.0 * i);
137                 SimpleNode s = new SimpleJavaNode(1);
138                 s.setScope(new SourceFileScope("foo"));
139                 s.testingOnly__setBeginLine(i);
140                 s.testingOnly__setBeginColumn(1);
141                 s.testingOnly__setBeginColumn(1);
142                 points[i].setNode(s);
143                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
144 
145                 lPoints.add(points[i]);
146             }
147 
148             Collections.shuffle(lPoints);
149             for (int i = 0; i < POINTS; i++) {
150                 IUT.addDataPoint((DataPoint) lPoints.get(i));
151             }
152         }
153 
154     }
155 
156     /***
157      * This test verifies that the Stat rule creates a Metric,
158      * with the proper values.
159      */
160     public void testMetrics() throws Throwable {
161         Report report = makeReport(IUT);
162         Iterator metrics = report.metrics();
163 
164         assertTrue(metrics.hasNext());
165         Object o = metrics.next();
166 
167         assertTrue(o instanceof Metric);
168         Metric m = (Metric) o;
169 
170         assertEquals("test.net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
171 
172         assertEquals(0.0, m.getLowValue(), 0.05);
173         assertEquals(POINTS - 1.0, m.getHighValue(), 0.05);
174         assertEquals(MEAN, m.getAverage(), 0.05);
175         assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
176     }
177 
178     /***
179      * This returns a Random value for Sigma which will
180      * return some values.
181      */
182     public double randomSigma() {
183         return random.nextDouble() * 1.0;
184     }
185 
186     /***
187      * This returns a Random value for Sigma which value
188      * is greater than the parameter.
189      */
190     public double randomSigma(int minimum) {
191         double minSigma = ((POINTS - 1 - minimum) - MEAN) / SIGMA;
192 
193         if ((minSigma <= 0) || (minSigma > 2))
194             return randomSigma();
195 
196         return minSigma + (random.nextDouble() * (2 - minSigma));
197     }
198 
199     /***
200      * This returns the expected number of results when
201      * the Sigma rating is the smallest.
202      */
203     public int expectedSigma(double sigma) {
204         long expectedMin = Math.round(MEAN + (sigma * SIGMA));
205 
206         if (((POINTS - 1) - expectedMin) < 0)
207             return 0;
208         return (POINTS - 1) - (int) expectedMin;
209     }
210 
211     /***
212      * This generates a random minimum value for testing.
213      */
214     public double randomMinimum() {
215         return random.nextDouble() * (POINTS - 1);
216     }
217 
218     /***
219      * This generates a random minimum value for which fewer
220      * results would be returned.
221      */
222     public double randomMinimum(int minimum) {
223         double diffTarget = 1.0 * (POINTS - 1 - minimum);
224         return (random.nextDouble() * minimum) + diffTarget;
225     }
226 
227     /***
228      * This returns the expected number of reports.
229      * <p/>
230      * If the Minimum comes in at 521.569 then we expect
231      * 522, 523, ... 999 will pass.
232      */
233     public int expectedMinimum(double minimum) {
234         Double d = new Double(minimum);
235         return POINTS - 1 - d.intValue();
236     }
237 
238     public void testExpectedMinimum() {
239         for (int i = 0; i < POINTS - 1; i++) {
240             assertEquals("Integer Min", POINTS - 1 - i, expectedMinimum(i * 1.0));
241             assertEquals("Double Min", POINTS - 1 - i, expectedMinimum((i * 1.0) + 0.5));
242         }
243     }
244 
245     /***
246      * This returns a random value for Top Score.
247      */
248     public int randomTopScore() {
249         return random.nextInt(POINTS - 1);
250     }
251 
252     /***
253      * This will return a random value for the Top Score
254      * which will return more than the minimum provided.
255      */
256     public int randomTopScore(double target) {
257         if (target < 0)
258             return 0;
259 
260         return random.nextInt((new Double(target)).intValue());
261     }
262 
263     /***
264      * This will return the expected number of results
265      * with the given Top Score.
266      */
267     public int expectedTopScore(int target) {
268         return target;
269     }
270 
271     // Test Single Datapoint
272     public void testSingleDatapoint() {
273         StatisticalRule IUT = new MockStatisticalRule();
274 
275         DataPoint point = new DataPoint();
276         point.setScore(POINTS + 1.0);
277         SimpleNode s = new SimpleJavaNode(1);
278         s.setScope(new SourceFileScope("foo"));
279         s.testingOnly__setBeginLine(POINTS + 1);
280         s.testingOnly__setBeginColumn(1);
281         point.setNode(s);
282         point.setMessage("SingleDataPoint");
283 
284         IUT.addProperty("minimum", Integer.toString(POINTS));
285 
286         IUT.addDataPoint(point);
287 
288         Report report = makeReport(IUT);
289 
290         assertEquals("Expecting only one result.", 1, report.size());
291     }
292 
293     // Okay, we have three properties we need to
294     // test in Combination:
295     //  S = Sigma
296     //  T = Top Score
297     //  M = Minimum
298     //
299     // They are listed in decreasing order of what
300     // to expect.
301     //
302     // Thus testSM() should have the Sigma less than
303     // the minimum, so we expect the Minimum # of results.
304     //
305 
306     public void testS() throws Throwable {
307         verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
308 
309         for (int i = 0; i < NUM_TESTS; i++) {
310             double sigma = randomSigma();
311             verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
312         }
313     }
314 
315     public void testS1() throws Throwable {
316         testS();
317     }
318 
319     public void testS2() throws Throwable {
320         testS();
321     }
322 
323     public void testS3() throws Throwable {
324         testS();
325     }
326 
327     public void testS4() throws Throwable {
328         testS();
329     }
330 
331     public void testS5() throws Throwable {
332         testS();
333     }
334 
335 
336     public void testT() throws Throwable {
337         verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
338 
339         for (int i = 0; i < NUM_TESTS; i++) {
340             int topScore = randomTopScore();
341             verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
342         }
343     }
344 
345     public void testT1() throws Throwable {
346         testT();
347     }
348 
349     public void testT2() throws Throwable {
350         testT();
351     }
352 
353     public void testT3() throws Throwable {
354         testT();
355     }
356 
357     public void testT4() throws Throwable {
358         testT();
359     }
360 
361     public void testT5() throws Throwable {
362         testT();
363     }
364 
365     public void testM() throws Throwable {
366         verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
367 
368         for (int i = 0; i < NUM_TESTS; i++) {
369             double minimum = randomMinimum();
370             verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
371         }
372     }
373 
374     public void testM1() throws Throwable {
375         testM();
376     }
377 
378     public void testM2() throws Throwable {
379         testM();
380     }
381 
382     public void testM3() throws Throwable {
383         testM();
384     }
385 
386     public void testM4() throws Throwable {
387         testM();
388     }
389 
390     public void testM5() throws Throwable {
391         testM();
392     }
393 
394     public void testST() throws Throwable {
395         verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
396 
397         for (int i = 0; i < NUM_TESTS; i++) {
398             double sigma = randomSigma();
399             int topScore = randomTopScore(expectedSigma(sigma));
400 
401             verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
402         }
403     }
404 
405     public void testST1() throws Throwable {
406         testST();
407     }
408 
409     public void testST2() throws Throwable {
410         testST();
411     }
412 
413     public void testST3() throws Throwable {
414         testST();
415     }
416 
417     public void testST4() throws Throwable {
418         testST();
419     }
420 
421     public void testST5() throws Throwable {
422         testST();
423     }
424 
425     public void testTS() throws Throwable {
426         verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
427 
428         for (int i = 0; i < NUM_TESTS; i++) {
429             int topScore = randomTopScore();
430             double sigma = randomSigma(expectedTopScore(topScore));
431 
432             verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
433         }
434     }
435 
436     public void testTS1() throws Throwable {
437         testTS();
438     }
439 
440     public void testTS2() throws Throwable {
441         testTS();
442     }
443 
444     public void testTS3() throws Throwable {
445         testTS();
446     }
447 
448     public void testTS4() throws Throwable {
449         testTS();
450     }
451 
452     public void testTS5() throws Throwable {
453         testTS();
454     }
455 
456     public void testSM() throws Throwable {
457         verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
458         for (int i = 0; i < NUM_TESTS; i++) {
459             double sigma = randomSigma();
460             double minimum = randomMinimum(expectedSigma(sigma));
461 
462             verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
463         }
464 
465     }
466 
467     public void testSM1() throws Throwable {
468         testSM();
469     }
470 
471     public void testSM2() throws Throwable {
472         testSM();
473     }
474 
475     public void testSM3() throws Throwable {
476         testSM();
477     }
478 
479     public void testSM4() throws Throwable {
480         testSM();
481     }
482 
483     public void testSM5() throws Throwable {
484         testSM();
485     }
486 
487 
488     public void testMS() throws Throwable {
489         verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
490         for (int i = 0; i < NUM_TESTS; i++) {
491             double minimum = randomMinimum();
492             double sigma = randomSigma(expectedMinimum(minimum));
493 
494             verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
495         }
496     }
497 
498     public void testMS1() throws Throwable {
499         testMS();
500     }
501 
502     public void testMS2() throws Throwable {
503         testMS();
504     }
505 
506     public void testMS3() throws Throwable {
507         testMS();
508     }
509 
510     public void testMS4() throws Throwable {
511         testMS();
512     }
513 
514     public void testMS5() throws Throwable {
515         testMS();
516     }
517 
518 
519     public void testTM() throws Throwable {
520         verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
521         for (int i = 0; i < NUM_TESTS; i++) {
522             int topScore = randomTopScore();
523             double minimum = randomMinimum(expectedTopScore(topScore));
524 
525             verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
526         }
527     }
528 
529     public void testTM1() throws Throwable {
530         testTM();
531     }
532 
533     public void testTM2() throws Throwable {
534         testTM();
535     }
536 
537     public void testTM3() throws Throwable {
538         testTM();
539     }
540 
541     public void testTM4() throws Throwable {
542         testTM();
543     }
544 
545     public void testTM5() throws Throwable {
546         testTM();
547     }
548 
549 
550     public void testMT() throws Throwable {
551         verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
552         for (int i = 0; i < NUM_TESTS; i++) {
553             double minimum = randomMinimum();
554             int topScore = randomTopScore(expectedMinimum(minimum));
555 
556             verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
557         }
558     }
559 
560     public void testMT1() throws Throwable {
561         testMT();
562     }
563 
564     public void testMT2() throws Throwable {
565         testMT();
566     }
567 
568     public void testMT3() throws Throwable {
569         testMT();
570     }
571 
572     public void testMT4() throws Throwable {
573         testMT();
574     }
575 
576     public void testMT5() throws Throwable {
577         testMT();
578     }
579 
580 
581     public void testSTM() throws Throwable {
582         double sigma = randomSigma();
583         verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
584 
585         for (int i = 0; i < NUM_TESTS; i++) {
586             sigma = randomSigma();
587             int topScore = randomTopScore(expectedSigma(sigma));
588             double minimum = randomMinimum(expectedTopScore(topScore));
589 
590             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
591         }
592     }
593 
594     public void testSTM1() throws Throwable {
595         testSTM();
596     }
597 
598     public void testSTM2() throws Throwable {
599         testSTM();
600     }
601 
602     public void testSTM3() throws Throwable {
603         testSTM();
604     }
605 
606     public void testSTM4() throws Throwable {
607         testSTM();
608     }
609 
610     public void testSTM5() throws Throwable {
611         testSTM();
612     }
613 
614     public void testSMT() throws Throwable {
615         double sigma = randomSigma();
616         verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
617 
618         for (int i = 0; i < NUM_TESTS; i++) {
619             sigma = randomSigma();
620             double minimum = randomMinimum(expectedSigma(sigma));
621             int topScore = randomTopScore(expectedMinimum(minimum));
622 
623             verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
624         }
625     }
626 
627     public void testSMT1() throws Throwable {
628         testSMT();
629     }
630 
631     public void testSMT2() throws Throwable {
632         testSMT();
633     }
634 
635     public void testSMT3() throws Throwable {
636         testSMT();
637     }
638 
639     public void testSMT4() throws Throwable {
640         testSMT();
641     }
642 
643     public void testSMT5() throws Throwable {
644         testSMT();
645     }
646 
647     public void testTSM() throws Throwable {
648         int topScore = randomTopScore();
649         verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
650 
651         for (int i = 0; i < NUM_TESTS; i++) {
652             topScore = randomTopScore();
653             double sigma = randomSigma(expectedTopScore(topScore));
654             double minimum = randomMinimum(expectedSigma(sigma));
655 
656             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
657         }
658     }
659 
660     public void testTSM1() throws Throwable {
661         testTSM();
662     }
663 
664     public void testTSM2() throws Throwable {
665         testTSM();
666     }
667 
668     public void testTSM3() throws Throwable {
669         testTSM();
670     }
671 
672     public void testTSM4() throws Throwable {
673         testTSM();
674     }
675 
676     public void testTSM5() throws Throwable {
677         testTSM();
678     }
679 
680     public void testTMS() throws Throwable {
681         int topScore = randomTopScore();
682         verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
683 
684         for (int i = 0; i < NUM_TESTS; i++) {
685             topScore = randomTopScore();
686             double minimum = randomMinimum(expectedTopScore(topScore));
687             double sigma = randomSigma(expectedMinimum(minimum));
688 
689             verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
690         }
691     }
692 
693     public void testTMS1() throws Throwable {
694         testTMS();
695     }
696 
697     public void testTMS2() throws Throwable {
698         testTMS();
699     }
700 
701     public void testTMS3() throws Throwable {
702         testTMS();
703     }
704 
705     public void testTMS4() throws Throwable {
706         testTMS();
707     }
708 
709     public void testTMS5() throws Throwable {
710         testTMS();
711     }
712 
713     /***
714      * Verifies what happens when you pass these parameters
715      * into the thing.  DELTA is the amount of error allowed.
716      * Usually DELTA is only used for Sigma, as we really can't
717      * calculate it exactly.
718      */
719 
720     public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
721         try {
722             setUp();
723             if (sigma >= 0) {
724                 IUT.addProperty("sigma", Double.toString(sigma));
725             }
726 
727             if (minimum >= 0) {
728                 IUT.addProperty("minimum", Double.toString(minimum));
729             }
730 
731             if (topScore >= 0) {
732                 IUT.addProperty("topscore", Integer.toString(topScore));
733             }
734 
735             Report report = makeReport(IUT);
736             if (delta == 0) {
737                 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
738             } else {
739                 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
740 
741                 assertTrue(assertStr, report.size() >= (expected - delta));
742                 assertTrue(assertStr, report.size() <= (expected + delta));
743             }
744         } catch (AssertionFailedError afe) {
745             System.err.println("******** " + testName + " ***********");
746             if (sigma != NO_SIGMA) {
747                 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
748             }
749 
750             if (minimum != NO_MINIMUM) {
751                 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
752             }
753 
754             if (topScore != NO_TOPSCORE) {
755                 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
756             }
757 
758             throw afe;
759 
760         }
761     }
762 
763     public Report makeReport(Rule IUT) {
764         List list = new ArrayList();
765         Report report = new Report();
766 
767         RuleContext ctx = new RuleContext();
768         ctx.setReport(report);
769         ctx.setSourceCodeFilename(testName);
770 
771         IUT.apply(list, ctx);
772 
773         return report;
774     }
775 }