1 /**
2 * Copyright 2003-2006 Greg Luck
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package net.sf.ehcache;
18
19 import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
20 import net.sf.ehcache.store.Store;
21 import net.sf.ehcache.store.LruMemoryStoreTest;
22 import net.sf.ehcache.distribution.JVMUtil;
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.Random;
30
31 /**
32 * Other than policy differences, the Store implementations should work identically
33 *
34 * @author Greg Luck
35 * @version $Id: MemoryStoreTester.java 53 2006-04-25 08:56:21Z gregluck $
36 */
37 public class MemoryStoreTester extends AbstractCacheTest {
38
39 private static final Log LOG = LogFactory.getLog(MemoryStoreTester.class.getName());
40
41 /**
42 * The memory store that tests will be performed on
43 */
44 protected Store store;
45
46 /**
47 * For automatic suite generators
48 */
49 public void testNoop() {
50
51 }
52
53 /**
54 * setup test
55 */
56 protected void setUp() throws Exception {
57 manager = CacheManager.getInstance();
58 }
59
60 /**
61 * teardown
62 */
63 protected void tearDown() throws Exception {
64 if (manager != null) {
65 manager.shutdown();
66 }
67 }
68
69 /**
70 * Creates a cache with the given policy and adds it to the manager.
71 *
72 * @param evictionPolicy
73 * @throws CacheException
74 */
75 protected void createMemoryStore(MemoryStoreEvictionPolicy evictionPolicy) throws CacheException {
76 Cache cache = new Cache("test", 1000, evictionPolicy, true, null, true, 60, 30, false, 60, null);
77 manager.addCache(cache);
78 store = cache.getMemoryStore();
79 }
80
81 /**
82 * Creates a cache with the given policy and adds it to the manager.
83 *
84 * @param evictionPolicy
85 * @throws CacheException
86 */
87 protected void createMemoryStore(MemoryStoreEvictionPolicy evictionPolicy, int memoryStoreSize) throws CacheException {
88 manager.removeCache("test");
89 Cache cache = new Cache("test", memoryStoreSize, evictionPolicy, true, null, true, 60, 30, false, 60, null);
90 manager.addCache(cache);
91 store = cache.getMemoryStore();
92 }
93
94 /**
95 * Creates a store from the given configuration and cache within it.
96 *
97 * @param filePath
98 * @param cacheName
99 * @throws CacheException
100 */
101 protected void createMemoryStore(String filePath, String cacheName) throws CacheException {
102 manager.shutdown();
103 manager = CacheManager.create(filePath);
104 Cache cache = manager.getCache(cacheName);
105 store = cache.getMemoryStore();
106 }
107
108
109 /**
110 * Test elements can be put in the store
111 */
112 protected void putTest() throws IOException {
113 Element element;
114
115 assertEquals(0, store.getSize());
116
117 element = new Element("key1", "value1");
118 store.put(element);
119 assertEquals(1, store.getSize());
120 assertEquals("value1", store.get("key1").getObjectValue());
121
122 element = new Element("key2", "value2");
123 store.put(element);
124 assertEquals(2, store.getSize());
125 assertEquals("value2", store.get("key2").getObjectValue());
126 }
127
128 /**
129 * Test elements can be removed from the store
130 */
131 protected void removeTest() throws IOException {
132 Element element;
133
134 element = new Element("key1", "value1");
135 store.put(element);
136 assertEquals(1, store.getSize());
137
138 store.remove("key1");
139 assertEquals(0, store.getSize());
140
141 store.put(new Element("key2", "value2"));
142 store.put(new Element("key3", "value3"));
143 assertEquals(2, store.getSize());
144
145 assertNotNull(store.remove("key2"));
146 assertEquals(1, store.getSize());
147
148
149 assertNull(store.remove("key4"));
150 assertEquals(1, store.getSize());
151
152
153 assertNull(store.remove(null));
154 }
155
156
157 /**
158 * Check no NPE on put
159 */
160 public void testNullPut() throws IOException {
161 store.put(null);
162 }
163
164 /**
165 * Check no NPE on get
166 */
167 public void testNullGet() throws IOException {
168 assertNull(store.get(null));
169 }
170
171 /**
172 * Check no NPE on remove
173 */
174 public void testNullRemove() throws IOException {
175 assertNull(store.remove(null));
176 }
177
178 /**
179 * Tests looking up an entry that does not exist.
180 */
181 public void testGetUnknown() throws Exception {
182 final Element element = store.get("key");
183 assertNull(element);
184 }
185
186 /**
187 * Tests adding an entry.
188 */
189 public void testPut() throws Exception {
190 final String value = "value";
191 final String key = "key";
192
193
194 assertEquals(0, store.getSize());
195 Element element = store.get(key);
196 assertNull(element);
197
198
199 element = new Element(key, value);
200 store.put(element);
201
202
203 assertEquals(1, store.getSize());
204 element = store.get(key);
205 assertNotNull(element);
206 assertEquals(value, element.getObjectValue());
207 }
208
209 /**
210 * Tests removing an entry.
211 */
212 public void testRemove() throws Exception {
213 final String value = "value";
214 final String key = "key";
215
216
217
218 Element element = new Element(key, value);
219 store.put(element);
220
221
222 assertEquals(1, store.getSize());
223 element = store.get(key);
224 assertNotNull(element);
225
226
227 store.remove(key);
228
229
230 assertEquals(0, store.getSize());
231 element = store.get(key);
232 assertNull(element);
233 }
234
235 /**
236 * Tests removing all the entries.
237 */
238 public void testRemoveAll() throws Exception {
239 final String value = "value";
240 final String key = "key";
241
242
243 Element element = new Element(key, value);
244 store.put(element);
245
246
247 element = store.get(key);
248 assertNotNull(element);
249
250
251 store.removeAll();
252
253
254 assertEquals(0, store.getSize());
255 element = store.get(key);
256 assertNull(element);
257 }
258
259 /**
260 * Tests bulk load.
261 */
262 public void testBulkLoad() throws Exception {
263 final Random random = new Random();
264 StopWatch stopWatch = new StopWatch();
265
266
267
268 for (int i = 0; i < 500; i++) {
269
270 final String key = "key" + i;
271 final String value = "value" + random.nextInt(1000);
272
273
274 Element element = new Element(key, value);
275 store.put(element);
276 element = store.get(key);
277 assertNotNull(element);
278
279
280 store.remove(key);
281 element = store.get(key);
282 assertNull(element);
283
284 element = new Element(key, value);
285 store.put(element);
286 element = store.get(key);
287 assertNotNull(element);
288 }
289 long time = stopWatch.getElapsedTime();
290 LOG.info("Time for Bulk Load: " + time);
291 }
292
293 /**
294 * Benchmark to test speed.
295 */
296 public void testBenchmarkPutGetRemove() throws Exception {
297 final String key = "key";
298 byte[] value = new byte[500];
299 StopWatch stopWatch = new StopWatch();
300
301
302 for (int i = 0; i < 50000; i++) {
303 Element element = new Element(key, value);
304 store.put(element);
305 store.get(key + i);
306 }
307 for (int i = 0; i < 50000; i++) {
308 store.remove(key + i);
309 }
310 long time = stopWatch.getElapsedTime();
311 LOG.info("Time for benchmarkPutGetRemove: " + time);
312 assertTrue("Too slow. Time was " + time, time < 500);
313 }
314
315 /**
316 * Benchmark to test speed.
317 */
318 public void testBenchmarkPutGet() throws Exception {
319 final String key = "key";
320 byte[] value = new byte[500];
321 StopWatch stopWatch = new StopWatch();
322
323
324 for (int i = 0; i < 50000; i++) {
325 Element element = new Element(key, value);
326 store.put(element);
327 }
328 for (int i = 0; i < 50000; i++) {
329 store.get(key + i);
330 }
331 long time = stopWatch.getElapsedTime();
332 LOG.info("Time for benchmarkPutGet: " + time);
333 assertTrue("Too slow. Time was " + time, time < 300);
334 }
335
336
337 /**
338 * Benchmark to test speed.
339 * Original implementation 12seconds
340 * This implementation 9 seconds
341 */
342 public void benchmarkPutGetSuryaTest(long allowedTime) throws Exception {
343 Random random = new Random();
344 byte[] value = new byte[500];
345 StopWatch stopWatch = new StopWatch();
346
347
348 for (int i = 0; i < 50000; i++) {
349 String key = "key" + i;
350
351 Element element = new Element(key, value);
352 store.put(element);
353
354
355 int accesses = random.nextInt(5);
356 for (int j = 0; j <= accesses; j++) {
357 store.get(key);
358 }
359 }
360 long time = stopWatch.getElapsedTime();
361 LOG.info("Time for benchmarkPutGetSurya: " + time);
362 assertTrue("Too slow. Time was " + time, time < allowedTime);
363 }
364
365 /**
366 * Multi-thread read-only test.
367 */
368 public void testReadOnlyThreads() throws Exception {
369
370
371 store.put(new Element("key0", "value"));
372 store.put(new Element("key1", "value"));
373
374
375 final List executables = new ArrayList();
376 for (int i = 0; i < 10; i++) {
377 final String key = "key" + (i % 2);
378 final MemoryStoreTester.Executable executable = new LruMemoryStoreTest.Executable() {
379 public void execute() throws Exception {
380 final Element element = store.get(key);
381 assertNotNull(element);
382 assertEquals("value", element.getObjectValue());
383 }
384 };
385 executables.add(executable);
386 }
387 runThreads(executables);
388 }
389
390 /**
391 * Multi-thread read-write test.
392 */
393 public void testReadWriteThreads() throws Exception {
394
395 final String value = "value";
396 final String key = "key";
397
398
399 final Element element = new Element(key, value);
400 store.put(element);
401
402
403 final List executables = new ArrayList();
404 for (int i = 0; i < 5; i++) {
405 final MemoryStoreTester.Executable executable = new MemoryStoreTester.Executable() {
406 public void execute() throws Exception {
407 final Element element = store.get("key");
408 assertNotNull(element);
409 }
410 };
411 executables.add(executable);
412 }
413 for (int i = 0; i < 5; i++) {
414 final MemoryStoreTester.Executable executable = new MemoryStoreTester.Executable() {
415 public void execute() throws Exception {
416 store.put(element);
417 }
418 };
419 executables.add(executable);
420 }
421
422 runThreads(executables);
423 }
424
425 /**
426 * Multi-thread read, put and removeAll test.
427 * This checks for memory leaks
428 * using the removeAll which was the known cause of memory leaks with LruMemoryStore in JCS
429 */
430 public void testMemoryLeak() throws Exception {
431 long differenceMemoryCache = thrashCache();
432 assertTrue(differenceMemoryCache < 500000);
433 }
434
435
436 /**
437 * This method tries to get the store too leak.
438 */
439 protected long thrashCache() throws Exception {
440
441
442 long startingSize = measureMemoryUse();
443 LOG.info("Memory Used is: " + startingSize);
444
445 final String value = "value";
446 final String key = "key";
447
448
449 final Element element = new Element(key, value);
450 store.put(element);
451
452
453 final List executables = new ArrayList();
454 for (int i = 0; i < 15; i++) {
455 final LruMemoryStoreTest.Executable executable = new MemoryStoreTester.Executable() {
456 public void execute() throws Exception {
457 for (int i = 0; i < 500; i++) {
458 final String key = "key" + i;
459 store.get(key);
460 }
461 store.get("key");
462 }
463 };
464 executables.add(executable);
465 }
466
467 for (int i = 0; i < 15; i++) {
468 final LruMemoryStoreTest.Executable executable = new MemoryStoreTester.Executable() {
469 public void execute() throws Exception {
470
471
472 for (int i = 0; i < 500; i++) {
473
474 final String key = "key" + i;
475 byte[] value = new byte[10000];
476 Element element = new Element(key, value);
477 store.put(element);
478 }
479 }
480 };
481 executables.add(executable);
482 }
483
484 runThreads(executables);
485 store.removeAll();
486
487 long finishingSize = measureMemoryUse();
488 LOG.info("Memory Used is: " + finishingSize);
489 return finishingSize - startingSize;
490 }
491
492
493 /**
494 * Multi-thread read-write test.
495 */
496 public void testReadWriteThreadsSurya() throws Exception {
497
498 long start = System.currentTimeMillis();
499 final List executables = new ArrayList();
500 final Random random = new Random();
501
502
503 for (int i = 0; i < 10; i++) {
504 final Executable executable = new Executable() {
505 public void execute() throws Exception {
506 store.get("key" + random.nextInt(10000));
507 }
508 };
509 executables.add(executable);
510 }
511
512
513 for (int i = 0; i < 5; i++) {
514 final Executable executable = new Executable() {
515 public void execute() throws Exception {
516 store.put(new Element("key" + random.nextInt(20000), "value"));
517 }
518 };
519 executables.add(executable);
520 }
521
522
523 for (int i = 0; i < 5; i++) {
524 final Executable executable = new Executable() {
525 public void execute() throws Exception {
526 store.remove("key" + random.nextInt(10000));
527 }
528 };
529 executables.add(executable);
530 }
531
532 runThreads(executables);
533 long end = System.currentTimeMillis();
534 LOG.info("Total time for the test: " + (end + start) + " ms");
535 }
536
537
538 /**
539 * Runs a set of threads, for a fixed amount of time.
540 */
541 protected void runThreads(final List executables) throws Exception {
542
543 final long endTime = System.currentTimeMillis() + 10000;
544 final Throwable[] errors = new Throwable[1];
545
546
547 final Thread[] threads = new Thread[executables.size()];
548 for (int i = 0; i < threads.length; i++) {
549 final LruMemoryStoreTest.Executable executable = (MemoryStoreTester.Executable) executables.get(i);
550 threads[i] = new Thread() {
551 public void run() {
552 try {
553
554 while (System.currentTimeMillis() < endTime) {
555 executable.execute();
556 }
557 } catch (Throwable t) {
558
559 errors[0] = t;
560 }
561 }
562 };
563 threads[i].start();
564 }
565
566
567 for (int i = 0; i < threads.length; i++) {
568 threads[i].join();
569 }
570
571
572 if (errors[0] != null) {
573 throw new Exception("Test thread failed.", errors[0]);
574 }
575 }
576
577
578 /**
579 * A runnable, that can throw an exception.
580 */
581 protected interface Executable {
582 /**
583 * Executes this object.
584 *
585 * @throws Exception
586 */
587 void execute() throws Exception;
588 }
589
590
591
592 /**
593 * Test behaviour of memory store using 1 million records.
594 * This is expected to run out of memory on a 64MB machine. Where it runs out
595 * is asserted so that design changes do not start using more memory per element.
596 * <p/>
597 * This test will fail (ie not get an out of memory error) on VMs configured to be server which do not have a fixed upper memory limit.
598 * <p/>
599 * Takes too long to run therefore switch off
600 * <p/>
601 * These memory size asserts were 100,000 and 60,000. The ApacheLRU map does not get quite as high numbers.
602 */
603 public void testMemoryStoreOutOfMemoryLimit() throws Exception {
604
605 Cache cache = new Cache("memoryLimitTest", 1000000, false, false, 500, 500);
606 manager.addCache(cache);
607 int i = 0;
608 try {
609 for (; i < 1000000; i++) {
610 cache.put(new Element("" +
611 i, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
612 + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
613 + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
614 + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
615 + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
616 + "AAAAA " + i));
617 }
618 fail();
619 } catch (OutOfMemoryError e) {
620 if (JVMUtil.isJDK15()) {
621 assertTrue(i > 90000);
622 } else {
623 assertTrue(i > 50000);
624 }
625 LOG.info("Ran out of memory putting " + i + "th element");
626 }
627 }
628
629 }