1 /***
2 * <copyright>
3 * Copyright 1997-2002 BBNT Solutions, LLC
4 * under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the Cougaar Open Source License as published by
8 * DARPA on the Cougaar Open Source Website (www.cougaar.org).
9 *
10 * THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11 * PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12 * IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14 * ANY WARRANTIES AS TO NON-INFRINGEMENT. IN NO EVENT SHALL COPYRIGHT
15 * HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17 * TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18 * PERFORMANCE OF THE COUGAAR SOFTWARE.
19 * </copyright>
20 *
21 * Created on Aug 26, 2002
22 */
23 package test.net.sourceforge.pmd.stat;
24
25 import junit.framework.AssertionFailedError;
26 import junit.framework.TestCase;
27 import net.sourceforge.pmd.Report;
28 import net.sourceforge.pmd.Rule;
29 import net.sourceforge.pmd.RuleContext;
30 import net.sourceforge.pmd.stat.DataPoint;
31 import net.sourceforge.pmd.stat.Metric;
32 import net.sourceforge.pmd.stat.StatisticalRule;
33
34 import java.util.ArrayList;
35 import java.util.Collections;
36 import java.util.Iterator;
37 import java.util.List;
38 import java.util.Random;
39
40 /***
41 * This class tests the Statistical Rules in PMD.
42 *
43 * The idea is, that we fill up 999 datapoints into
44 * the Stat Rule, and then throw random parameters
45 * at it.
46 *
47 * The three parameters which are checked are:
48 * sigma - # Sigmas over the mean.
49 * topscore - Only the top 5 or so items.
50 * minimum - Only things of score 10 or better
51 *
52 * When more than one parameter is lumped together, then
53 * we expect the one which would return the fewest to
54 * determine what gets sent back.
55 *
56 * So, we throw each collection of parameters, where each
57 * one is a different order into the system. We check the
58 * results off of what the smallest value should be.
59 *
60 * If you are going to work with StatisticalRule any, please
61 * bump the "NUM_TESTS" number up to something like 128. That
62 * way you are more likely to identify problems. It is set low
63 * now to make building and running tests easier (when we aren't
64 * touching the file.)
65 *
66 * Note also, that when verifying the Sigma, I wasn't quite able
67 * to determine how many results it would return (it would vary
68 * from -2 to 2 of what I expected.) That is what the delta
69 * parameter on the verify method takes. If you can figure it
70 * out exactly, (without stealing code from the StatRule) then
71 * feel free to change it and tighten the deltas.
72 */
73 public class StatisticalRuleTest extends TestCase {
74
75 private static final int POINTS = 100;
76
77 private DataPoint points[] = new DataPoint[POINTS];
78 private MockStatisticalRule IUT = null;
79 private String testName = null;
80 private Random random = new Random();
81
82 public static final double MAX_MINIMUM = POINTS;
83 public static final double NO_MINIMUM = -1.0;
84 public static final double MAX_SIGMA = 5.0;
85 public static final double NO_SIGMA = -1.0;
86 public static final int MIN_TOPSCORE = 0;
87 public static final int NO_TOPSCORE = -1;
88
89
90 public static final double MEAN = 49.5;
91 public static final double SIGMA = 29.0115;
92 public static final int NUM_TESTS = 1;
93
94 public static final double DELTA = 0.005;
95
96 public StatisticalRuleTest(String name) {
97 super(name);
98 this.testName = name;
99 }
100
101 public void setUp() {
102 IUT = new MockStatisticalRule();
103 if (testName.endsWith("0")) {
104 for (int i = 0; i < POINTS; i++) {
105 points[i] = new DataPoint();
106 points[i].setScore(1.0 * i);
107 points[i].setLineNumber(i);
108 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
109
110 IUT.addDataPoint(points[i]);
111 }
112 } else if (testName.endsWith("1")) {
113 for (int i = POINTS-1; i >= 0; i--) {
114 points[i] = new DataPoint();
115 points[i].setScore(1.0 * i);
116 points[i].setLineNumber(i);
117 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
118
119 IUT.addDataPoint(points[i]);
120 }
121 } else {
122 List lPoints = new ArrayList();
123 for (int i = 0; i < POINTS; i++) {
124 DataPoint point = new DataPoint();
125 point.setScore(1.0 * i);
126 point.setLineNumber(i);
127 point.setMessage("DataPoint[" + Integer.toString(i) + "]");
128
129 lPoints.add(point);
130 }
131
132 Collections.shuffle(lPoints);
133 for (int i = 0; i < POINTS; i++) {
134 IUT.addDataPoint((DataPoint) lPoints.get(i));
135 }
136 }
137
138 }
139
140 /***
141 * This test verifies that the Stat rule creates a Metric,
142 * with the proper values.
143 */
144 public void testMetrics() throws Throwable {
145 Report report = makeReport(IUT);
146 Iterator metrics = report.metrics();
147
148 assertTrue(metrics.hasNext());
149 Object o = metrics.next();
150
151 assertTrue(o instanceof Metric);
152 Metric m = (Metric) o;
153
154 assertEquals("test.net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
155
156 assertEquals(0.0, m.getLowValue(), 0.05);
157 assertEquals(POINTS -1.0, m.getHighValue(), 0.05);
158 assertEquals(MEAN, m.getAverage(), 0.05);
159 assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
160 }
161
162 /***
163 * This returns a Random value for Sigma which will
164 * return some values.
165 */
166 public double randomSigma() {
167 return random.nextDouble() * 1.0;
168 }
169
170 /***
171 * This returns a Random value for Sigma which value
172 * is greater than the parameter.
173 */
174 public double randomSigma(int minimum) {
175 double minSigma = ((POINTS -1 - minimum) - MEAN) / SIGMA;
176
177 if ((minSigma <= 0) || (minSigma > 2))
178 return randomSigma();
179
180 return minSigma + (random.nextDouble() * (2 - minSigma));
181 }
182
183 /***
184 * This returns the expected number of results when
185 * the Sigma rating is the smallest.
186 */
187 public int expectedSigma(double sigma) {
188 long expectedMin = Math.round(MEAN + (sigma * SIGMA));
189
190 if (((POINTS -1) - expectedMin) < 0)
191 return 0;
192 return (POINTS -1) - (int) expectedMin;
193 }
194
195 /***
196 * This generates a random minimum value for testing.
197 */
198 public double randomMinimum() {
199 return random.nextDouble() * (POINTS -1);
200 }
201
202 /***
203 * This generates a random minimum value for which fewer
204 * results would be returned.
205 */
206 public double randomMinimum(int minimum) {
207 double diffTarget = 1.0 * (POINTS -1 - minimum);
208 return (random.nextDouble() * minimum) + diffTarget;
209 }
210
211 /***
212 * This returns the expected number of reports.
213 *
214 * If the Minimum comes in at 521.569 then we expect
215 * 522, 523, ... 999 will pass.
216 */
217 public int expectedMinimum(double minimum) {
218 Double d = new Double(minimum);
219 return POINTS -1 - d.intValue();
220 }
221
222 public void testExpectedMinimum() {
223 for (int i = 0; i < POINTS -1; i++) {
224 assertEquals("Integer Min", POINTS -1 - i, expectedMinimum(i * 1.0));
225 assertEquals("Double Min", POINTS -1 - i, expectedMinimum((i * 1.0) + 0.5));
226 }
227 }
228
229 /***
230 * This returns a random value for Top Score.
231 */
232 public int randomTopScore() {
233 return random.nextInt(POINTS -1);
234 }
235
236 /***
237 * This will return a random value for the Top Score
238 * which will return more than the minimum provided.
239 */
240 public int randomTopScore(double target) {
241 if (target < 0)
242 return 0;
243
244 return random.nextInt((new Double(target)).intValue());
245 }
246
247 /***
248 * This will return the expected number of results
249 * with the given Top Score.
250 */
251 public int expectedTopScore(int target) {
252 return target;
253 }
254
255 // Test Single Datapoint
256 public void testSingleDatapoint() {
257 StatisticalRule IUT = new MockStatisticalRule();
258
259 DataPoint point = new DataPoint();
260 point.setScore(POINTS + 1.0);
261 point.setLineNumber(POINTS + 1);
262 point.setMessage("SingleDataPoint");
263
264 IUT.addProperty("minimum", Integer.toString(POINTS));
265
266 IUT.addDataPoint(point);
267
268 Report report = makeReport(IUT);
269
270 assertEquals("Expecting only one result.", 1, report.size());
271 }
272
273 // Okay, we have three properties we need to
274 // test in Combination:
275 // S = Sigma
276 // T = Top Score
277 // M = Minimum
278 //
279 // They are listed in decreasing order of what
280 // to expect.
281 //
282 // Thus testSM() should have the Sigma less than
283 // the minimum, so we expect the Minimum # of results.
284 //
285
286 public void testS() throws Throwable {
287 verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
288
289 for (int i = 0; i < NUM_TESTS; i++) {
290 double sigma = randomSigma();
291 verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
292 }
293 }
294
295 public void testS1() throws Throwable {
296 testS();
297 }
298
299 public void testS2() throws Throwable {
300 testS();
301 }
302
303 public void testS3() throws Throwable {
304 testS();
305 }
306
307 public void testS4() throws Throwable {
308 testS();
309 }
310
311 public void testS5() throws Throwable {
312 testS();
313 }
314
315
316 public void testT() throws Throwable {
317 verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
318
319 for (int i = 0; i < NUM_TESTS; i++) {
320 int topScore = randomTopScore();
321 verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
322 }
323 }
324
325 public void testT1() throws Throwable {
326 testT();
327 }
328
329 public void testT2() throws Throwable {
330 testT();
331 }
332
333 public void testT3() throws Throwable {
334 testT();
335 }
336
337 public void testT4() throws Throwable {
338 testT();
339 }
340
341 public void testT5() throws Throwable {
342 testT();
343 }
344
345 public void testM() throws Throwable {
346 verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
347
348 for (int i = 0; i < NUM_TESTS; i++) {
349 double minimum = randomMinimum();
350 verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
351 }
352 }
353
354 public void testM1() throws Throwable {
355 testM();
356 }
357
358 public void testM2() throws Throwable {
359 testM();
360 }
361
362 public void testM3() throws Throwable {
363 testM();
364 }
365
366 public void testM4() throws Throwable {
367 testM();
368 }
369
370 public void testM5() throws Throwable {
371 testM();
372 }
373
374 public void testST() throws Throwable {
375 verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
376
377 for (int i = 0; i < NUM_TESTS; i++) {
378 double sigma = randomSigma();
379 int topScore = randomTopScore(expectedSigma(sigma));
380
381 verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
382 }
383 }
384
385 public void testST1() throws Throwable {
386 testST();
387 }
388
389 public void testST2() throws Throwable {
390 testST();
391 }
392
393 public void testST3() throws Throwable {
394 testST();
395 }
396
397 public void testST4() throws Throwable {
398 testST();
399 }
400
401 public void testST5() throws Throwable {
402 testST();
403 }
404
405 public void testTS() throws Throwable {
406 verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
407
408 for (int i = 0; i < NUM_TESTS; i++) {
409 int topScore = randomTopScore();
410 double sigma = randomSigma(expectedTopScore(topScore));
411
412 verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
413 }
414 }
415
416 public void testTS1() throws Throwable {
417 testTS();
418 }
419
420 public void testTS2() throws Throwable {
421 testTS();
422 }
423
424 public void testTS3() throws Throwable {
425 testTS();
426 }
427
428 public void testTS4() throws Throwable {
429 testTS();
430 }
431
432 public void testTS5() throws Throwable {
433 testTS();
434 }
435
436 public void testSM() throws Throwable {
437 verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
438 for (int i = 0; i < NUM_TESTS; i++) {
439 double sigma = randomSigma();
440 double minimum = randomMinimum(expectedSigma(sigma));
441
442 verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
443 }
444
445 }
446
447 public void testSM1() throws Throwable {
448 testSM();
449 }
450
451 public void testSM2() throws Throwable {
452 testSM();
453 }
454
455 public void testSM3() throws Throwable {
456 testSM();
457 }
458
459 public void testSM4() throws Throwable {
460 testSM();
461 }
462
463 public void testSM5() throws Throwable {
464 testSM();
465 }
466
467
468 public void testMS() throws Throwable {
469 verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
470 for (int i = 0; i < NUM_TESTS; i++) {
471 double minimum = randomMinimum();
472 double sigma = randomSigma(expectedMinimum(minimum));
473
474 verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
475 }
476 }
477
478 public void testMS1() throws Throwable {
479 testMS();
480 }
481
482 public void testMS2() throws Throwable {
483 testMS();
484 }
485
486 public void testMS3() throws Throwable {
487 testMS();
488 }
489
490 public void testMS4() throws Throwable {
491 testMS();
492 }
493
494 public void testMS5() throws Throwable {
495 testMS();
496 }
497
498
499 public void testTM() throws Throwable {
500 verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
501 for (int i = 0; i < NUM_TESTS; i++) {
502 int topScore = randomTopScore();
503 double minimum = randomMinimum(expectedTopScore(topScore));
504
505 verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
506 }
507 }
508
509 public void testTM1() throws Throwable {
510 testTM();
511 }
512
513 public void testTM2() throws Throwable {
514 testTM();
515 }
516
517 public void testTM3() throws Throwable {
518 testTM();
519 }
520
521 public void testTM4() throws Throwable {
522 testTM();
523 }
524
525 public void testTM5() throws Throwable {
526 testTM();
527 }
528
529
530 public void testMT() throws Throwable {
531 verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
532 for (int i = 0; i < NUM_TESTS; i++) {
533 double minimum = randomMinimum();
534 int topScore = randomTopScore(expectedMinimum(minimum));
535
536 verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
537 }
538 }
539
540 public void testMT1() throws Throwable {
541 testMT();
542 }
543
544 public void testMT2() throws Throwable {
545 testMT();
546 }
547
548 public void testMT3() throws Throwable {
549 testMT();
550 }
551
552 public void testMT4() throws Throwable {
553 testMT();
554 }
555
556 public void testMT5() throws Throwable {
557 testMT();
558 }
559
560
561 public void testSTM() throws Throwable {
562 double sigma = randomSigma();
563 verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
564
565 for (int i = 0; i < NUM_TESTS; i++) {
566 sigma = randomSigma();
567 int topScore = randomTopScore(expectedSigma(sigma));
568 double minimum = randomMinimum(expectedTopScore(topScore));
569
570 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
571 }
572 }
573
574 public void testSTM1() throws Throwable {
575 testSTM();
576 }
577
578 public void testSTM2() throws Throwable {
579 testSTM();
580 }
581
582 public void testSTM3() throws Throwable {
583 testSTM();
584 }
585
586 public void testSTM4() throws Throwable {
587 testSTM();
588 }
589
590 public void testSTM5() throws Throwable {
591 testSTM();
592 }
593
594 public void testSMT() throws Throwable {
595 double sigma = randomSigma();
596 verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
597
598 for (int i = 0; i < NUM_TESTS; i++) {
599 sigma = randomSigma();
600 double minimum = randomMinimum(expectedSigma(sigma));
601 int topScore = randomTopScore(expectedMinimum(minimum));
602
603 verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
604 }
605 }
606
607 public void testSMT1() throws Throwable {
608 testSMT();
609 }
610
611 public void testSMT2() throws Throwable {
612 testSMT();
613 }
614
615 public void testSMT3() throws Throwable {
616 testSMT();
617 }
618
619 public void testSMT4() throws Throwable {
620 testSMT();
621 }
622
623 public void testSMT5() throws Throwable {
624 testSMT();
625 }
626
627 public void testTSM() throws Throwable {
628 int topScore = randomTopScore();
629 verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
630
631 for (int i = 0; i < NUM_TESTS; i++) {
632 topScore = randomTopScore();
633 double sigma = randomSigma(expectedTopScore(topScore));
634 double minimum = randomMinimum(expectedSigma(sigma));
635
636 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
637 }
638 }
639
640 public void testTSM1() throws Throwable {
641 testTSM();
642 }
643
644 public void testTSM2() throws Throwable {
645 testTSM();
646 }
647
648 public void testTSM3() throws Throwable {
649 testTSM();
650 }
651
652 public void testTSM4() throws Throwable {
653 testTSM();
654 }
655
656 public void testTSM5() throws Throwable {
657 testTSM();
658 }
659
660 public void testTMS() throws Throwable {
661 int topScore = randomTopScore();
662 verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
663
664 for (int i = 0; i < NUM_TESTS; i++) {
665 topScore = randomTopScore();
666 double minimum = randomMinimum(expectedTopScore(topScore));
667 double sigma = randomSigma(expectedMinimum(minimum));
668
669 verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
670 }
671 }
672
673 public void testTMS1() throws Throwable {
674 testTMS();
675 }
676
677 public void testTMS2() throws Throwable {
678 testTMS();
679 }
680
681 public void testTMS3() throws Throwable {
682 testTMS();
683 }
684
685 public void testTMS4() throws Throwable {
686 testTMS();
687 }
688
689 public void testTMS5() throws Throwable {
690 testTMS();
691 }
692
693 /***
694 * Verifies what happens when you pass these parameters
695 * into the thing. DELTA is the amount of error allowed.
696 * Usually DELTA is only used for Sigma, as we really can't
697 * calculate it exactly.
698 */
699
700 public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
701 try {
702 setUp();
703 if (sigma >= 0) {
704 IUT.addProperty("sigma", Double.toString(sigma));
705 }
706
707 if (minimum >= 0) {
708 IUT.addProperty("minimum", Double.toString(minimum));
709 }
710
711 if (topScore >= 0) {
712 IUT.addProperty("topscore", Integer.toString(topScore));
713 }
714
715 Report report = makeReport(IUT);
716 if (delta == 0) {
717 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
718 } else {
719 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
720
721 assertTrue(assertStr, report.size() >= (expected - delta));
722 assertTrue(assertStr, report.size() <= (expected + delta));
723 }
724 } catch (AssertionFailedError afe) {
725 System.err.println("******** " + testName + " ***********");
726 if (sigma != NO_SIGMA) {
727 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
728 }
729
730 if (minimum != NO_MINIMUM) {
731 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
732 }
733
734 if (topScore != NO_TOPSCORE) {
735 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
736 }
737
738 throw afe;
739
740 }
741 }
742
743 public Report makeReport(Rule IUT) {
744 List list = new ArrayList();
745 Report report = new Report();
746
747 RuleContext ctx = new RuleContext();
748 ctx.setReport(report);
749 ctx.setSourceCodeFilename(testName);
750
751 IUT.apply(list, ctx);
752
753 return report;
754 }
755 }
This page was automatically generated by Maven