1 /*
2 * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /*
25 * Common code for string deduplication tests
26 */
27
28 import java.lang.management.*;
29 import java.lang.reflect.*;
30 import java.security.*;
31 import java.util.*;
32 import com.oracle.java.testlibrary.*;
33 import sun.misc.*;
34
35 class TestStringDeduplicationTools {
36 private static final String YoungGC = "YoungGC";
37 private static final String FullGC = "FullGC";
38
39 private static final int Xmn = 50; // MB
40 private static final int Xms = 100; // MB
41 private static final int Xmx = 100; // MB
42 private static final int MB = 1024 * 1024;
43 private static final int StringLength = 50;
44
45 private static Field valueField;
46 private static Unsafe unsafe;
47 private static byte[] dummy;
48
49 static {
50 try {
51 Field field = Unsafe.class.getDeclaredField("theUnsafe");
52 field.setAccessible(true);
53 unsafe = (Unsafe)field.get(null);
54
55 valueField = String.class.getDeclaredField("value");
56 valueField.setAccessible(true);
57 } catch (Exception e) {
58 throw new RuntimeException(e);
59 }
60 }
61
62 private static Object getValue(String string) {
63 try {
64 return valueField.get(string);
65 } catch (Exception e) {
66 throw new RuntimeException(e);
67 }
68 }
69
70 private static void doFullGc(int numberOfTimes) {
71 for (int i = 0; i < numberOfTimes; i++) {
72 System.out.println("Begin: Full GC " + (i + 1) + "/" + numberOfTimes);
73 System.gc();
74 System.out.println("End: Full GC " + (i + 1) + "/" + numberOfTimes);
75 }
76 }
77
78 private static void doYoungGc(int numberOfTimes) {
79 // Provoke at least numberOfTimes young GCs
80 final int objectSize = 128;
81 final int maxObjectInYoung = (Xmn * MB) / objectSize;
82 for (int i = 0; i < numberOfTimes; i++) {
83 System.out.println("Begin: Young GC " + (i + 1) + "/" + numberOfTimes);
84 for (int j = 0; j < maxObjectInYoung + 1; j++) {
85 dummy = new byte[objectSize];
86 }
87 System.out.println("End: Young GC " + (i + 1) + "/" + numberOfTimes);
88 }
89 }
90
91 private static void forceDeduplication(int ageThreshold, String gcType) {
92 // Force deduplication to happen by either causing a FullGC or a YoungGC.
93 // We do several collections to also provoke a situation where the the
94 // deduplication thread needs to yield while processing the queue. This
95 // also tests that the references in the deduplication queue are adjusted
96 // accordingly.
97 if (gcType.equals(FullGC)) {
98 doFullGc(3);
99 } else {
100 doYoungGc(ageThreshold + 3);
101 }
102 }
103
104 private static String generateString(int id) {
105 StringBuilder builder = new StringBuilder(StringLength);
106
107 builder.append("DeduplicationTestString:" + id + ":");
108
109 while (builder.length() < StringLength) {
110 builder.append('X');
111 }
112
113 return builder.toString();
114 }
115
116 private static ArrayList<String> createStrings(int total, int unique) {
117 System.out.println("Creating strings: total=" + total + ", unique=" + unique);
118 if (total % unique != 0) {
119 throw new RuntimeException("Total must be divisible by unique");
120 }
121
122 ArrayList<String> list = new ArrayList<String>(total);
123 for (int j = 0; j < total / unique; j++) {
124 for (int i = 0; i < unique; i++) {
125 list.add(generateString(i));
126 }
127 }
128
129 return list;
130 }
131
132 private static void verifyStrings(ArrayList<String> list, int uniqueExpected) {
133 for (;;) {
134 // Check number of deduplicated strings
135 ArrayList<Object> unique = new ArrayList<Object>(uniqueExpected);
136 for (String string: list) {
137 Object value = getValue(string);
138 boolean uniqueValue = true;
139 for (Object obj: unique) {
140 if (obj == value) {
141 uniqueValue = false;
142 break;
143 }
144 }
145
146 if (uniqueValue) {
147 unique.add(value);
148 }
149 }
150
151 System.out.println("Verifying strings: total=" + list.size() +
152 ", uniqueFound=" + unique.size() +
153 ", uniqueExpected=" + uniqueExpected);
154
155 if (unique.size() == uniqueExpected) {
156 System.out.println("Deduplication completed");
157 break;
158 } else {
159 System.out.println("Deduplication not completed, waiting...");
160
161 // Give the deduplication thread time to complete
162 try {
163 Thread.sleep(1000);
164 } catch (Exception e) {
165 throw new RuntimeException(e);
166 }
167 }
168 }
169 }
170
171 private static OutputAnalyzer runTest(String... extraArgs) throws Exception {
172 String[] defaultArgs = new String[] {
173 "-Xmn" + Xmn + "m",
174 "-Xms" + Xms + "m",
175 "-Xmx" + Xmx + "m",
176 "-XX:+UseG1GC",
177 "-XX:+UnlockDiagnosticVMOptions",
178 "-XX:+VerifyAfterGC" // Always verify after GC
179 };
180
181 ArrayList<String> args = new ArrayList<String>();
182 args.addAll(Arrays.asList(defaultArgs));
183 args.addAll(Arrays.asList(extraArgs));
184
185 ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(args.toArray(new String[args.size()]));
186 OutputAnalyzer output = new OutputAnalyzer(pb.start());
187 System.err.println(output.getStderr());
188 System.out.println(output.getStdout());
189 return output;
190 }
191
192 private static class DeduplicationTest {
193 public static void main(String[] args) {
194 System.out.println("Begin: DeduplicationTest");
195
196 final int numberOfStrings = Integer.parseUnsignedInt(args[0]);
197 final int numberOfUniqueStrings = Integer.parseUnsignedInt(args[1]);
198 final int ageThreshold = Integer.parseUnsignedInt(args[2]);
199 final String gcType = args[3];
200
201 ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings);
202 forceDeduplication(ageThreshold, gcType);
203 verifyStrings(list, numberOfUniqueStrings);
204
205 System.out.println("End: DeduplicationTest");
206 }
207
208 public static OutputAnalyzer run(int numberOfStrings, int ageThreshold, String gcType, String... extraArgs) throws Exception {
209 String[] defaultArgs = new String[] {
210 "-XX:+UseStringDeduplication",
211 "-XX:StringDeduplicationAgeThreshold=" + ageThreshold,
212 DeduplicationTest.class.getName(),
213 "" + numberOfStrings,
214 "" + numberOfStrings / 2,
215 "" + ageThreshold,
216 gcType
217 };
218
219 ArrayList<String> args = new ArrayList<String>();
220 args.addAll(Arrays.asList(extraArgs));
221 args.addAll(Arrays.asList(defaultArgs));
222
223 return runTest(args.toArray(new String[args.size()]));
224 }
225 }
226
227 private static class InternedTest {
228 public static void main(String[] args) {
229 // This test verifies that interned strings are always
230 // deduplicated when being interned, and never after
231 // being interned.
232
233 System.out.println("Begin: InternedTest");
234
235 final int ageThreshold = Integer.parseUnsignedInt(args[0]);
236 final String baseString = "DeduplicationTestString:" + InternedTest.class.getName();
237
238 // Create duplicate of baseString
239 StringBuilder sb1 = new StringBuilder(baseString);
240 String dupString1 = sb1.toString();
241 if (getValue(dupString1) == getValue(baseString)) {
242 throw new RuntimeException("Values should not match");
243 }
244
245 // Force baseString to be inspected for deduplication
246 // and be inserted into the deduplication hashtable.
247 forceDeduplication(ageThreshold, FullGC);
248
249 // Wait for deduplication to occur
250 while (getValue(dupString1) != getValue(baseString)) {
251 System.out.println("Waiting...");
252 try {
253 Thread.sleep(100);
254 } catch (Exception e) {
255 throw new RuntimeException(e);
256 }
257 }
258
259 // Create a new duplicate of baseString
260 StringBuilder sb2 = new StringBuilder(baseString);
261 String dupString2 = sb2.toString();
262 if (getValue(dupString2) == getValue(baseString)) {
263 throw new RuntimeException("Values should not match");
264 }
265
266 // Intern the new duplicate
267 Object beforeInternedValue = getValue(dupString2);
268 String internedString = dupString2.intern();
269 if (internedString != dupString2) {
270 throw new RuntimeException("String should match");
271 }
272 if (getValue(internedString) != getValue(baseString)) {
273 throw new RuntimeException("Values should match");
274 }
275
276 // Check original value of interned string, to make sure
277 // deduplication happened on the interned string and not
278 // on the base string
279 if (beforeInternedValue == getValue(baseString)) {
280 throw new RuntimeException("Values should not match");
281 }
282
283 System.out.println("End: InternedTest");
284 }
285
286 public static OutputAnalyzer run() throws Exception {
287 return runTest("-XX:+PrintGC",
288 "-XX:+PrintGCDetails",
289 "-XX:+UseStringDeduplication",
290 "-XX:+PrintStringDeduplicationStatistics",
291 "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold,
292 InternedTest.class.getName(),
293 "" + DefaultAgeThreshold);
294 }
295 }
296
297 private static class MemoryUsageTest {
298 public static void main(String[] args) {
299 System.out.println("Begin: MemoryUsageTest");
300
301 final boolean useStringDeduplication = Boolean.parseBoolean(args[0]);
302 final int numberOfStrings = LargeNumberOfStrings;
303 final int numberOfUniqueStrings = 1;
304
305 ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings);
306 forceDeduplication(DefaultAgeThreshold, FullGC);
307
308 if (useStringDeduplication) {
309 verifyStrings(list, numberOfUniqueStrings);
310 }
311
312 System.gc();
313
314 System.out.println("Heap Memory Usage: " + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed());
315 System.out.println("Array Header Size: " + unsafe.ARRAY_CHAR_BASE_OFFSET);
316
317 System.out.println("End: MemoryUsageTest");
318 }
319
320 public static OutputAnalyzer run(boolean useStringDeduplication) throws Exception {
321 String[] extraArgs = new String[0];
322
323 if (useStringDeduplication) {
324 extraArgs = new String[] {
325 "-XX:+UseStringDeduplication",
326 "-XX:+PrintStringDeduplicationStatistics",
327 "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold
328 };
329 }
330
331 String[] defaultArgs = new String[] {
332 "-XX:+PrintGC",
333 "-XX:+PrintGCDetails",
334 MemoryUsageTest.class.getName(),
335 "" + useStringDeduplication
336 };
337
338 ArrayList<String> args = new ArrayList<String>();
339 args.addAll(Arrays.asList(extraArgs));
340 args.addAll(Arrays.asList(defaultArgs));
341
342 return runTest(args.toArray(new String[args.size()]));
343 }
344 }
345
346 /*
347 * Tests
348 */
349
350 private static final int LargeNumberOfStrings = 10000;
351 private static final int SmallNumberOfStrings = 10;
352
353 private static final int MaxAgeThreshold = 15;
354 private static final int DefaultAgeThreshold = 3;
355 private static final int MinAgeThreshold = 1;
356
357 private static final int TooLowAgeThreshold = MinAgeThreshold - 1;
358 private static final int TooHighAgeThreshold = MaxAgeThreshold + 1;
359
360 public static void testYoungGC() throws Exception {
361 // Do young GC to age strings to provoke deduplication
362 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
363 DefaultAgeThreshold,
364 YoungGC,
365 "-XX:+PrintGC",
366 "-XX:+PrintStringDeduplicationStatistics");
367 output.shouldNotContain("Full GC");
368 output.shouldContain("GC pause (G1 Evacuation Pause) (young)");
369 output.shouldContain("GC concurrent-string-deduplication");
370 output.shouldContain("Deduplicated:");
371 output.shouldHaveExitValue(0);
372 }
373
374 public static void testFullGC() throws Exception {
375 // Do full GC to age strings to provoke deduplication
376 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
377 DefaultAgeThreshold,
378 FullGC,
379 "-XX:+PrintGC",
380 "-XX:+PrintStringDeduplicationStatistics");
381 output.shouldNotContain("GC pause (G1 Evacuation Pause) (young)");
382 output.shouldContain("Full GC");
383 output.shouldContain("GC concurrent-string-deduplication");
384 output.shouldContain("Deduplicated:");
385 output.shouldHaveExitValue(0);
386 }
387
388 public static void testTableResize() throws Exception {
389 // Test with StringDeduplicationResizeALot
390 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
391 DefaultAgeThreshold,
392 YoungGC,
393 "-XX:+PrintGC",
394 "-XX:+PrintStringDeduplicationStatistics",
395 "-XX:+StringDeduplicationResizeALot");
396 output.shouldContain("GC concurrent-string-deduplication");
397 output.shouldContain("Deduplicated:");
398 output.shouldNotContain("Resize Count: 0");
399 output.shouldHaveExitValue(0);
400 }
401
402 public static void testTableRehash() throws Exception {
403 // Test with StringDeduplicationRehashALot
404 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
405 DefaultAgeThreshold,
406 YoungGC,
407 "-XX:+PrintGC",
408 "-XX:+PrintStringDeduplicationStatistics",
409 "-XX:+StringDeduplicationRehashALot");
410 output.shouldContain("GC concurrent-string-deduplication");
411 output.shouldContain("Deduplicated:");
412 output.shouldNotContain("Rehash Count: 0");
413 output.shouldNotContain("Hash Seed: 0x0");
414 output.shouldHaveExitValue(0);
415 }
416
417 public static void testAgeThreshold() throws Exception {
418 OutputAnalyzer output;
419
420 // Test with max age theshold
421 output = DeduplicationTest.run(SmallNumberOfStrings,
422 MaxAgeThreshold,
423 YoungGC,
424 "-XX:+PrintGC",
425 "-XX:+PrintStringDeduplicationStatistics");
426 output.shouldContain("GC concurrent-string-deduplication");
427 output.shouldContain("Deduplicated:");
428 output.shouldHaveExitValue(0);
429
430 // Test with min age theshold
431 output = DeduplicationTest.run(SmallNumberOfStrings,
432 MinAgeThreshold,
433 YoungGC,
434 "-XX:+PrintGC",
435 "-XX:+PrintStringDeduplicationStatistics");
436 output.shouldContain("GC concurrent-string-deduplication");
437 output.shouldContain("Deduplicated:");
438 output.shouldHaveExitValue(0);
439
440 // Test with too low age threshold
441 output = DeduplicationTest.run(SmallNumberOfStrings,
442 TooLowAgeThreshold,
443 YoungGC);
444 output.shouldContain("StringDeduplicationAgeThreshold of " + TooLowAgeThreshold +
445 " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold);
446 output.shouldHaveExitValue(1);
447
448 // Test with too high age threshold
449 output = DeduplicationTest.run(SmallNumberOfStrings,
450 TooHighAgeThreshold,
451 YoungGC);
452 output.shouldContain("StringDeduplicationAgeThreshold of " + TooHighAgeThreshold +
453 " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold);
454 output.shouldHaveExitValue(1);
455 }
456
457 public static void testPrintOptions() throws Exception {
458 OutputAnalyzer output;
459
460 // Test without PrintGC and without PrintStringDeduplicationStatistics
461 output = DeduplicationTest.run(SmallNumberOfStrings,
462 DefaultAgeThreshold,
463 YoungGC);
464 output.shouldNotContain("GC concurrent-string-deduplication");
465 output.shouldNotContain("Deduplicated:");
466 output.shouldHaveExitValue(0);
467
468 // Test with PrintGC but without PrintStringDeduplicationStatistics
469 output = DeduplicationTest.run(SmallNumberOfStrings,
470 DefaultAgeThreshold,
471 YoungGC,
472 "-XX:+PrintGC");
473 output.shouldContain("GC concurrent-string-deduplication");
474 output.shouldNotContain("Deduplicated:");
475 output.shouldHaveExitValue(0);
476 }
477
478 public static void testInterned() throws Exception {
479 // Test that interned strings are deduplicated before being interned
480 OutputAnalyzer output = InternedTest.run();
481 output.shouldHaveExitValue(0);
482 }
483
484 public static void testMemoryUsage() throws Exception {
485 // Test that memory usage is reduced after deduplication
486 OutputAnalyzer output;
487 final String heapMemoryUsagePattern = "Heap Memory Usage: (\\d+)";
488 final String arrayHeaderSizePattern = "Array Header Size: (\\d+)";
489
490 // Run without deduplication
491 output = MemoryUsageTest.run(false);
492 output.shouldHaveExitValue(0);
493 final long heapMemoryUsageWithoutDedup = Long.parseLong(output.firstMatch(heapMemoryUsagePattern, 1));
494 final long arrayHeaderSizeWithoutDedup = Long.parseLong(output.firstMatch(arrayHeaderSizePattern, 1));
495
496 // Run with deduplication
497 output = MemoryUsageTest.run(true);
498 output.shouldHaveExitValue(0);
499 final long heapMemoryUsageWithDedup = Long.parseLong(output.firstMatch(heapMemoryUsagePattern, 1));
500 final long arrayHeaderSizeWithDedup = Long.parseLong(output.firstMatch(arrayHeaderSizePattern, 1));
501
502 // Sanity check to make sure one instance isn't using compressed class pointers and the other not
503 if (arrayHeaderSizeWithoutDedup != arrayHeaderSizeWithDedup) {
504 throw new Exception("Unexpected difference between array header sizes");
505 }
506
507 // Calculate expected memory usage with deduplication enabled. This calculation does
508 // not take alignment and padding into account, so it's a conservative estimate.
509 final long sizeOfChar = unsafe.ARRAY_CHAR_INDEX_SCALE;
510 final long sizeOfCharArray = StringLength * sizeOfChar + arrayHeaderSizeWithoutDedup;
511 final long bytesSaved = (LargeNumberOfStrings - 1) * sizeOfCharArray;
512 final long heapMemoryUsageWithDedupExpected = heapMemoryUsageWithoutDedup - bytesSaved;
513
514 System.out.println("Memory usage summary:");
515 System.out.println(" heapMemoryUsageWithoutDedup: " + heapMemoryUsageWithoutDedup);
516 System.out.println(" heapMemoryUsageWithDedup: " + heapMemoryUsageWithDedup);
517 System.out.println(" heapMemoryUsageWithDedupExpected: " + heapMemoryUsageWithDedupExpected);
518
519 if (heapMemoryUsageWithDedup > heapMemoryUsageWithDedupExpected) {
520 throw new Exception("Unexpected memory usage, heapMemoryUsageWithDedup should be less or equal to heapMemoryUsageWithDedupExpected");
521 }
522 }
523 }