1 /***
2 * <copyright>
3 * Copyright 1997-2002 BBNT Solutions, LLC
4 * under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the Cougaar Open Source License as published by
8 * DARPA on the Cougaar Open Source Website (www.cougaar.org).
9 *
10 * THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11 * PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12 * IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14 * ANY WARRANTIES AS TO NON-INFRINGEMENT. IN NO EVENT SHALL COPYRIGHT
15 * HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17 * TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18 * PERFORMANCE OF THE COUGAAR SOFTWARE.
19 * </copyright>
20 *
21 * Created on Aug 26, 2002
22 */
23 package test.net.sourceforge.pmd.stat;
24
25 import junit.framework.AssertionFailedError;
26 import junit.framework.TestCase;
27 import net.sourceforge.pmd.Report;
28 import net.sourceforge.pmd.Rule;
29 import net.sourceforge.pmd.RuleContext;
30 import net.sourceforge.pmd.ast.SimpleJavaNode;
31 import net.sourceforge.pmd.ast.SimpleNode;
32 import net.sourceforge.pmd.stat.DataPoint;
33 import net.sourceforge.pmd.stat.Metric;
34 import net.sourceforge.pmd.stat.StatisticalRule;
35 import net.sourceforge.pmd.symboltable.SourceFileScope;
36
37 import java.util.ArrayList;
38 import java.util.Collections;
39 import java.util.Iterator;
40 import java.util.List;
41 import java.util.Random;
42
43 /***
44 * This class tests the Statistical Rules in PMD.
45 * <p/>
46 * The idea is, that we fill up 999 datapoints into
47 * the Stat Rule, and then throw random parameters
48 * at it.
49 * <p/>
50 * The three parameters which are checked are:
51 * sigma - # Sigmas over the mean.
52 * topscore - Only the top 5 or so items.
53 * minimum - Only things of score 10 or better
54 * <p/>
55 * When more than one parameter is lumped together, then
56 * we expect the one which would return the fewest to
57 * determine what gets sent back.
58 * <p/>
59 * So, we throw each collection of parameters, where each
60 * one is a different order into the system. We check the
61 * results off of what the smallest value should be.
62 * <p/>
63 * If you are going to work with StatisticalRule any, please
64 * bump the "NUM_TESTS" number up to something like 128. That
65 * way you are more likely to identify problems. It is set low
66 * now to make building and running tests easier (when we aren't
67 * touching the file.)
68 * <p/>
69 * Note also, that when verifying the Sigma, I wasn't quite able
70 * to determine how many results it would return (it would vary
71 * from -2 to 2 of what I expected.) That is what the delta
72 * parameter on the verify method takes. If you can figure it
73 * out exactly, (without stealing code from the StatRule) then
74 * feel free to change it and tighten the deltas.
75 */
76 public class StatisticalRuleTest extends TestCase {
77
78 private static final int POINTS = 100;
79
80 private DataPoint points[] = new DataPoint[POINTS];
81 private MockStatisticalRule IUT = null;
82 private String testName = null;
83 private Random random = new Random();
84
85 public static final double MAX_MINIMUM = POINTS;
86 public static final double NO_MINIMUM = -1.0;
87 public static final double MAX_SIGMA = 5.0;
88 public static final double NO_SIGMA = -1.0;
89 public static final int MIN_TOPSCORE = 0;
90 public static final int NO_TOPSCORE = -1;
91
92
93 public static final double MEAN = 49.5;
94 public static final double SIGMA = 29.0115;
95 public static final int NUM_TESTS = 1;
96
97 public static final double DELTA = 0.005;
98
99 public StatisticalRuleTest(String name) {
100 super(name);
101 this.testName = name;
102 }
103
104 public void setUp() {
105 IUT = new MockStatisticalRule();
106 if (testName.endsWith("0")) {
107 for (int i = 0; i < POINTS; i++) {
108 points[i] = new DataPoint();
109 points[i].setScore(1.0 * i);
110 SimpleNode s = new SimpleJavaNode(1);
111 s.setScope(new SourceFileScope("foo"));
112 s.testingOnly__setBeginLine(i);
113 s.testingOnly__setBeginColumn(1);
114 points[i].setNode(s);
115 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
116
117 IUT.addDataPoint(points[i]);
118 }
119 } else if (testName.endsWith("1")) {
120 for (int i = POINTS - 1; i >= 0; i--) {
121 points[i] = new DataPoint();
122 points[i].setScore(1.0 * i);
123 SimpleNode s = new SimpleJavaNode(1);
124 s.setScope(new SourceFileScope("foo"));
125 s.testingOnly__setBeginLine(i);
126 s.testingOnly__setBeginColumn(1);
127 points[i].setNode(s);
128 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
129
130 IUT.addDataPoint(points[i]);
131 }
132 } else {
133 List lPoints = new ArrayList();
134 for (int i = 0; i < POINTS; i++) {
135 points[i] = new DataPoint();
136 points[i].setScore(1.0 * i);
137 SimpleNode s = new SimpleJavaNode(1);
138 s.setScope(new SourceFileScope("foo"));
139 s.testingOnly__setBeginLine(i);
140 s.testingOnly__setBeginColumn(1);
141 s.testingOnly__setBeginColumn(1);
142 points[i].setNode(s);
143 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
144
145 lPoints.add(points[i]);
146 }
147
148 Collections.shuffle(lPoints);
149 for (int i = 0; i < POINTS; i++) {
150 IUT.addDataPoint((DataPoint) lPoints.get(i));
151 }
152 }
153
154 }
155
156 /***
157 * This test verifies that the Stat rule creates a Metric,
158 * with the proper values.
159 */
160 public void testMetrics() throws Throwable {
161 Report report = makeReport(IUT);
162 Iterator metrics = report.metrics();
163
164 assertTrue(metrics.hasNext());
165 Object o = metrics.next();
166
167 assertTrue(o instanceof Metric);
168 Metric m = (Metric) o;
169
170 assertEquals("test.net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
171
172 assertEquals(0.0, m.getLowValue(), 0.05);
173 assertEquals(POINTS - 1.0, m.getHighValue(), 0.05);
174 assertEquals(MEAN, m.getAverage(), 0.05);
175 assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
176 }
177
178 /***
179 * This returns a Random value for Sigma which will
180 * return some values.
181 */
182 public double randomSigma() {
183 return random.nextDouble() * 1.0;
184 }
185
186 /***
187 * This returns a Random value for Sigma which value
188 * is greater than the parameter.
189 */
190 public double randomSigma(int minimum) {
191 double minSigma = ((POINTS - 1 - minimum) - MEAN) / SIGMA;
192
193 if ((minSigma <= 0) || (minSigma > 2))
194 return randomSigma();
195
196 return minSigma + (random.nextDouble() * (2 - minSigma));
197 }
198
199 /***
200 * This returns the expected number of results when
201 * the Sigma rating is the smallest.
202 */
203 public int expectedSigma(double sigma) {
204 long expectedMin = Math.round(MEAN + (sigma * SIGMA));
205
206 if (((POINTS - 1) - expectedMin) < 0)
207 return 0;
208 return (POINTS - 1) - (int) expectedMin;
209 }
210
211 /***
212 * This generates a random minimum value for testing.
213 */
214 public double randomMinimum() {
215 return random.nextDouble() * (POINTS - 1);
216 }
217
218 /***
219 * This generates a random minimum value for which fewer
220 * results would be returned.
221 */
222 public double randomMinimum(int minimum) {
223 double diffTarget = 1.0 * (POINTS - 1 - minimum);
224 return (random.nextDouble() * minimum) + diffTarget;
225 }
226
227 /***
228 * This returns the expected number of reports.
229 * <p/>
230 * If the Minimum comes in at 521.569 then we expect
231 * 522, 523, ... 999 will pass.
232 */
233 public int expectedMinimum(double minimum) {
234 Double d = new Double(minimum);
235 return POINTS - 1 - d.intValue();
236 }
237
238 public void testExpectedMinimum() {
239 for (int i = 0; i < POINTS - 1; i++) {
240 assertEquals("Integer Min", POINTS - 1 - i, expectedMinimum(i * 1.0));
241 assertEquals("Double Min", POINTS - 1 - i, expectedMinimum((i * 1.0) + 0.5));
242 }
243 }
244
245 /***
246 * This returns a random value for Top Score.
247 */
248 public int randomTopScore() {
249 return random.nextInt(POINTS - 1);
250 }
251
252 /***
253 * This will return a random value for the Top Score
254 * which will return more than the minimum provided.
255 */
256 public int randomTopScore(double target) {
257 if (target < 0)
258 return 0;
259
260 return random.nextInt((new Double(target)).intValue());
261 }
262
263 /***
264 * This will return the expected number of results
265 * with the given Top Score.
266 */
267 public int expectedTopScore(int target) {
268 return target;
269 }
270
271
272 public void testSingleDatapoint() {
273 StatisticalRule IUT = new MockStatisticalRule();
274
275 DataPoint point = new DataPoint();
276 point.setScore(POINTS + 1.0);
277 SimpleNode s = new SimpleJavaNode(1);
278 s.setScope(new SourceFileScope("foo"));
279 s.testingOnly__setBeginLine(POINTS + 1);
280 s.testingOnly__setBeginColumn(1);
281 point.setNode(s);
282 point.setMessage("SingleDataPoint");
283
284 IUT.addProperty("minimum", Integer.toString(POINTS));
285
286 IUT.addDataPoint(point);
287
288 Report report = makeReport(IUT);
289
290 assertEquals("Expecting only one result.", 1, report.size());
291 }
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306 public void testS() throws Throwable {
307 verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
308
309 for (int i = 0; i < NUM_TESTS; i++) {
310 double sigma = randomSigma();
311 verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
312 }
313 }
314
315 public void testS1() throws Throwable {
316 testS();
317 }
318
319 public void testS2() throws Throwable {
320 testS();
321 }
322
323 public void testS3() throws Throwable {
324 testS();
325 }
326
327 public void testS4() throws Throwable {
328 testS();
329 }
330
331 public void testS5() throws Throwable {
332 testS();
333 }
334
335
336 public void testT() throws Throwable {
337 verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
338
339 for (int i = 0; i < NUM_TESTS; i++) {
340 int topScore = randomTopScore();
341 verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
342 }
343 }
344
345 public void testT1() throws Throwable {
346 testT();
347 }
348
349 public void testT2() throws Throwable {
350 testT();
351 }
352
353 public void testT3() throws Throwable {
354 testT();
355 }
356
357 public void testT4() throws Throwable {
358 testT();
359 }
360
361 public void testT5() throws Throwable {
362 testT();
363 }
364
365 public void testM() throws Throwable {
366 verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
367
368 for (int i = 0; i < NUM_TESTS; i++) {
369 double minimum = randomMinimum();
370 verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
371 }
372 }
373
374 public void testM1() throws Throwable {
375 testM();
376 }
377
378 public void testM2() throws Throwable {
379 testM();
380 }
381
382 public void testM3() throws Throwable {
383 testM();
384 }
385
386 public void testM4() throws Throwable {
387 testM();
388 }
389
390 public void testM5() throws Throwable {
391 testM();
392 }
393
394 public void testST() throws Throwable {
395 verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
396
397 for (int i = 0; i < NUM_TESTS; i++) {
398 double sigma = randomSigma();
399 int topScore = randomTopScore(expectedSigma(sigma));
400
401 verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
402 }
403 }
404
405 public void testST1() throws Throwable {
406 testST();
407 }
408
409 public void testST2() throws Throwable {
410 testST();
411 }
412
413 public void testST3() throws Throwable {
414 testST();
415 }
416
417 public void testST4() throws Throwable {
418 testST();
419 }
420
421 public void testST5() throws Throwable {
422 testST();
423 }
424
425 public void testTS() throws Throwable {
426 verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
427
428 for (int i = 0; i < NUM_TESTS; i++) {
429 int topScore = randomTopScore();
430 double sigma = randomSigma(expectedTopScore(topScore));
431
432 verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
433 }
434 }
435
436 public void testTS1() throws Throwable {
437 testTS();
438 }
439
440 public void testTS2() throws Throwable {
441 testTS();
442 }
443
444 public void testTS3() throws Throwable {
445 testTS();
446 }
447
448 public void testTS4() throws Throwable {
449 testTS();
450 }
451
452 public void testTS5() throws Throwable {
453 testTS();
454 }
455
456 public void testSM() throws Throwable {
457 verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
458 for (int i = 0; i < NUM_TESTS; i++) {
459 double sigma = randomSigma();
460 double minimum = randomMinimum(expectedSigma(sigma));
461
462 verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
463 }
464
465 }
466
467 public void testSM1() throws Throwable {
468 testSM();
469 }
470
471 public void testSM2() throws Throwable {
472 testSM();
473 }
474
475 public void testSM3() throws Throwable {
476 testSM();
477 }
478
479 public void testSM4() throws Throwable {
480 testSM();
481 }
482
483 public void testSM5() throws Throwable {
484 testSM();
485 }
486
487
488 public void testMS() throws Throwable {
489 verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
490 for (int i = 0; i < NUM_TESTS; i++) {
491 double minimum = randomMinimum();
492 double sigma = randomSigma(expectedMinimum(minimum));
493
494 verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
495 }
496 }
497
498 public void testMS1() throws Throwable {
499 testMS();
500 }
501
502 public void testMS2() throws Throwable {
503 testMS();
504 }
505
506 public void testMS3() throws Throwable {
507 testMS();
508 }
509
510 public void testMS4() throws Throwable {
511 testMS();
512 }
513
514 public void testMS5() throws Throwable {
515 testMS();
516 }
517
518
519 public void testTM() throws Throwable {
520 verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
521 for (int i = 0; i < NUM_TESTS; i++) {
522 int topScore = randomTopScore();
523 double minimum = randomMinimum(expectedTopScore(topScore));
524
525 verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
526 }
527 }
528
529 public void testTM1() throws Throwable {
530 testTM();
531 }
532
533 public void testTM2() throws Throwable {
534 testTM();
535 }
536
537 public void testTM3() throws Throwable {
538 testTM();
539 }
540
541 public void testTM4() throws Throwable {
542 testTM();
543 }
544
545 public void testTM5() throws Throwable {
546 testTM();
547 }
548
549
550 public void testMT() throws Throwable {
551 verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
552 for (int i = 0; i < NUM_TESTS; i++) {
553 double minimum = randomMinimum();
554 int topScore = randomTopScore(expectedMinimum(minimum));
555
556 verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
557 }
558 }
559
560 public void testMT1() throws Throwable {
561 testMT();
562 }
563
564 public void testMT2() throws Throwable {
565 testMT();
566 }
567
568 public void testMT3() throws Throwable {
569 testMT();
570 }
571
572 public void testMT4() throws Throwable {
573 testMT();
574 }
575
576 public void testMT5() throws Throwable {
577 testMT();
578 }
579
580
581 public void testSTM() throws Throwable {
582 double sigma = randomSigma();
583 verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
584
585 for (int i = 0; i < NUM_TESTS; i++) {
586 sigma = randomSigma();
587 int topScore = randomTopScore(expectedSigma(sigma));
588 double minimum = randomMinimum(expectedTopScore(topScore));
589
590 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
591 }
592 }
593
594 public void testSTM1() throws Throwable {
595 testSTM();
596 }
597
598 public void testSTM2() throws Throwable {
599 testSTM();
600 }
601
602 public void testSTM3() throws Throwable {
603 testSTM();
604 }
605
606 public void testSTM4() throws Throwable {
607 testSTM();
608 }
609
610 public void testSTM5() throws Throwable {
611 testSTM();
612 }
613
614 public void testSMT() throws Throwable {
615 double sigma = randomSigma();
616 verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
617
618 for (int i = 0; i < NUM_TESTS; i++) {
619 sigma = randomSigma();
620 double minimum = randomMinimum(expectedSigma(sigma));
621 int topScore = randomTopScore(expectedMinimum(minimum));
622
623 verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
624 }
625 }
626
627 public void testSMT1() throws Throwable {
628 testSMT();
629 }
630
631 public void testSMT2() throws Throwable {
632 testSMT();
633 }
634
635 public void testSMT3() throws Throwable {
636 testSMT();
637 }
638
639 public void testSMT4() throws Throwable {
640 testSMT();
641 }
642
643 public void testSMT5() throws Throwable {
644 testSMT();
645 }
646
647 public void testTSM() throws Throwable {
648 int topScore = randomTopScore();
649 verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
650
651 for (int i = 0; i < NUM_TESTS; i++) {
652 topScore = randomTopScore();
653 double sigma = randomSigma(expectedTopScore(topScore));
654 double minimum = randomMinimum(expectedSigma(sigma));
655
656 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
657 }
658 }
659
660 public void testTSM1() throws Throwable {
661 testTSM();
662 }
663
664 public void testTSM2() throws Throwable {
665 testTSM();
666 }
667
668 public void testTSM3() throws Throwable {
669 testTSM();
670 }
671
672 public void testTSM4() throws Throwable {
673 testTSM();
674 }
675
676 public void testTSM5() throws Throwable {
677 testTSM();
678 }
679
680 public void testTMS() throws Throwable {
681 int topScore = randomTopScore();
682 verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
683
684 for (int i = 0; i < NUM_TESTS; i++) {
685 topScore = randomTopScore();
686 double minimum = randomMinimum(expectedTopScore(topScore));
687 double sigma = randomSigma(expectedMinimum(minimum));
688
689 verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
690 }
691 }
692
693 public void testTMS1() throws Throwable {
694 testTMS();
695 }
696
697 public void testTMS2() throws Throwable {
698 testTMS();
699 }
700
701 public void testTMS3() throws Throwable {
702 testTMS();
703 }
704
705 public void testTMS4() throws Throwable {
706 testTMS();
707 }
708
709 public void testTMS5() throws Throwable {
710 testTMS();
711 }
712
713 /***
714 * Verifies what happens when you pass these parameters
715 * into the thing. DELTA is the amount of error allowed.
716 * Usually DELTA is only used for Sigma, as we really can't
717 * calculate it exactly.
718 */
719
720 public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
721 try {
722 setUp();
723 if (sigma >= 0) {
724 IUT.addProperty("sigma", Double.toString(sigma));
725 }
726
727 if (minimum >= 0) {
728 IUT.addProperty("minimum", Double.toString(minimum));
729 }
730
731 if (topScore >= 0) {
732 IUT.addProperty("topscore", Integer.toString(topScore));
733 }
734
735 Report report = makeReport(IUT);
736 if (delta == 0) {
737 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
738 } else {
739 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
740
741 assertTrue(assertStr, report.size() >= (expected - delta));
742 assertTrue(assertStr, report.size() <= (expected + delta));
743 }
744 } catch (AssertionFailedError afe) {
745 System.err.println("******** " + testName + " ***********");
746 if (sigma != NO_SIGMA) {
747 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
748 }
749
750 if (minimum != NO_MINIMUM) {
751 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
752 }
753
754 if (topScore != NO_TOPSCORE) {
755 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
756 }
757
758 throw afe;
759
760 }
761 }
762
763 public Report makeReport(Rule IUT) {
764 List list = new ArrayList();
765 Report report = new Report();
766
767 RuleContext ctx = new RuleContext();
768 ctx.setReport(report);
769 ctx.setSourceCodeFilename(testName);
770
771 IUT.apply(list, ctx);
772
773 return report;
774 }
775 }