1 /* 2 * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 /* 25 * Common code for string deduplication tests 26 */ 27 28 import java.lang.management.*; 29 import java.lang.reflect.*; 30 import java.security.*; 31 import java.util.*; 32 import com.oracle.java.testlibrary.*; 33 import sun.misc.*; 34 35 class TestStringDeduplicationTools { 36 private static final String YoungGC = "YoungGC"; 37 private static final String FullGC = "FullGC"; 38 39 private static final int Xmn = 50; // MB 40 private static final int Xms = 100; // MB 41 private static final int Xmx = 100; // MB 42 private static final int MB = 1024 * 1024; 43 private static final int StringLength = 50; 44 45 private static Field valueField; 46 private static Unsafe unsafe; 47 private static byte[] dummy; 48 49 static { 50 try { 51 Field field = Unsafe.class.getDeclaredField("theUnsafe"); 52 field.setAccessible(true); 53 unsafe = (Unsafe)field.get(null); 54 55 valueField = String.class.getDeclaredField("value"); 56 valueField.setAccessible(true); 57 } catch (Exception e) { 58 throw new RuntimeException(e); 59 } 60 } 61 62 private static Object getValue(String string) { 63 try { 64 return valueField.get(string); 65 } catch (Exception e) { 66 throw new RuntimeException(e); 67 } 68 } 69 70 private static void doFullGc(int numberOfTimes) { 71 for (int i = 0; i < numberOfTimes; i++) { 72 System.out.println("Begin: Full GC " + (i + 1) + "/" + numberOfTimes); 73 System.gc(); 74 System.out.println("End: Full GC " + (i + 1) + "/" + numberOfTimes); 75 } 76 } 77 78 private static void doYoungGc(int numberOfTimes) { 79 // Provoke at least numberOfTimes young GCs 80 final int objectSize = 128; 81 final int maxObjectInYoung = (Xmn * MB) / objectSize; 82 for (int i = 0; i < numberOfTimes; i++) { 83 System.out.println("Begin: Young GC " + (i + 1) + "/" + numberOfTimes); 84 for (int j = 0; j < maxObjectInYoung + 1; j++) { 85 dummy = new byte[objectSize]; 86 } 87 System.out.println("End: Young GC " + (i + 1) + "/" + numberOfTimes); 88 } 89 } 90 91 private static void forceDeduplication(int ageThreshold, String gcType) { 92 // Force deduplication to happen by either causing a FullGC or a YoungGC. 93 // We do several collections to also provoke a situation where the the 94 // deduplication thread needs to yield while processing the queue. This 95 // also tests that the references in the deduplication queue are adjusted 96 // accordingly. 97 if (gcType.equals(FullGC)) { 98 doFullGc(3); 99 } else { 100 doYoungGc(ageThreshold + 3); 101 } 102 } 103 104 private static String generateString(int id) { 105 StringBuilder builder = new StringBuilder(StringLength); 106 107 builder.append("DeduplicationTestString:" + id + ":"); 108 109 while (builder.length() < StringLength) { 110 builder.append('X'); 111 } 112 113 return builder.toString(); 114 } 115 116 private static ArrayList<String> createStrings(int total, int unique) { 117 System.out.println("Creating strings: total=" + total + ", unique=" + unique); 118 if (total % unique != 0) { 119 throw new RuntimeException("Total must be divisible by unique"); 120 } 121 122 ArrayList<String> list = new ArrayList<String>(total); 123 for (int j = 0; j < total / unique; j++) { 124 for (int i = 0; i < unique; i++) { 125 list.add(generateString(i)); 126 } 127 } 128 129 return list; 130 } 131 132 private static void verifyStrings(ArrayList<String> list, int uniqueExpected) { 133 for (;;) { 134 // Check number of deduplicated strings 135 ArrayList<Object> unique = new ArrayList<Object>(uniqueExpected); 136 for (String string: list) { 137 Object value = getValue(string); 138 boolean uniqueValue = true; 139 for (Object obj: unique) { 140 if (obj == value) { 141 uniqueValue = false; 142 break; 143 } 144 } 145 146 if (uniqueValue) { 147 unique.add(value); 148 } 149 } 150 151 System.out.println("Verifying strings: total=" + list.size() + 152 ", uniqueFound=" + unique.size() + 153 ", uniqueExpected=" + uniqueExpected); 154 155 if (unique.size() == uniqueExpected) { 156 System.out.println("Deduplication completed"); 157 break; 158 } else { 159 System.out.println("Deduplication not completed, waiting..."); 160 161 // Give the deduplication thread time to complete 162 try { 163 Thread.sleep(1000); 164 } catch (Exception e) { 165 throw new RuntimeException(e); 166 } 167 } 168 } 169 } 170 171 private static OutputAnalyzer runTest(String... extraArgs) throws Exception { 172 String[] defaultArgs = new String[] { 173 "-Xmn" + Xmn + "m", 174 "-Xms" + Xms + "m", 175 "-Xmx" + Xmx + "m", 176 "-XX:+UseG1GC", 177 "-XX:+UnlockDiagnosticVMOptions", 178 "-XX:+VerifyAfterGC" // Always verify after GC 179 }; 180 181 ArrayList<String> args = new ArrayList<String>(); 182 args.addAll(Arrays.asList(defaultArgs)); 183 args.addAll(Arrays.asList(extraArgs)); 184 185 ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(args.toArray(new String[args.size()])); 186 OutputAnalyzer output = new OutputAnalyzer(pb.start()); 187 System.err.println(output.getStderr()); 188 System.out.println(output.getStdout()); 189 return output; 190 } 191 192 private static class DeduplicationTest { 193 public static void main(String[] args) { 194 System.out.println("Begin: DeduplicationTest"); 195 196 final int numberOfStrings = Integer.parseUnsignedInt(args[0]); 197 final int numberOfUniqueStrings = Integer.parseUnsignedInt(args[1]); 198 final int ageThreshold = Integer.parseUnsignedInt(args[2]); 199 final String gcType = args[3]; 200 201 ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings); 202 forceDeduplication(ageThreshold, gcType); 203 verifyStrings(list, numberOfUniqueStrings); 204 205 System.out.println("End: DeduplicationTest"); 206 } 207 208 public static OutputAnalyzer run(int numberOfStrings, int ageThreshold, String gcType, String... extraArgs) throws Exception { 209 String[] defaultArgs = new String[] { 210 "-XX:+UseStringDeduplication", 211 "-XX:StringDeduplicationAgeThreshold=" + ageThreshold, 212 DeduplicationTest.class.getName(), 213 "" + numberOfStrings, 214 "" + numberOfStrings / 2, 215 "" + ageThreshold, 216 gcType 217 }; 218 219 ArrayList<String> args = new ArrayList<String>(); 220 args.addAll(Arrays.asList(extraArgs)); 221 args.addAll(Arrays.asList(defaultArgs)); 222 223 return runTest(args.toArray(new String[args.size()])); 224 } 225 } 226 227 private static class InternedTest { 228 public static void main(String[] args) { 229 // This test verifies that interned strings are always 230 // deduplicated when being interned, and never after 231 // being interned. 232 233 System.out.println("Begin: InternedTest"); 234 235 final int ageThreshold = Integer.parseUnsignedInt(args[0]); 236 final String baseString = "DeduplicationTestString:" + InternedTest.class.getName(); 237 238 // Create duplicate of baseString 239 StringBuilder sb1 = new StringBuilder(baseString); 240 String dupString1 = sb1.toString(); 241 if (getValue(dupString1) == getValue(baseString)) { 242 throw new RuntimeException("Values should not match"); 243 } 244 245 // Force baseString to be inspected for deduplication 246 // and be inserted into the deduplication hashtable. 247 forceDeduplication(ageThreshold, FullGC); 248 249 // Wait for deduplication to occur 250 while (getValue(dupString1) != getValue(baseString)) { 251 System.out.println("Waiting..."); 252 try { 253 Thread.sleep(100); 254 } catch (Exception e) { 255 throw new RuntimeException(e); 256 } 257 } 258 259 // Create a new duplicate of baseString 260 StringBuilder sb2 = new StringBuilder(baseString); 261 String dupString2 = sb2.toString(); 262 if (getValue(dupString2) == getValue(baseString)) { 263 throw new RuntimeException("Values should not match"); 264 } 265 266 // Intern the new duplicate 267 Object beforeInternedValue = getValue(dupString2); 268 String internedString = dupString2.intern(); 269 if (internedString != dupString2) { 270 throw new RuntimeException("String should match"); 271 } 272 if (getValue(internedString) != getValue(baseString)) { 273 throw new RuntimeException("Values should match"); 274 } 275 276 // Check original value of interned string, to make sure 277 // deduplication happened on the interned string and not 278 // on the base string 279 if (beforeInternedValue == getValue(baseString)) { 280 throw new RuntimeException("Values should not match"); 281 } 282 283 System.out.println("End: InternedTest"); 284 } 285 286 public static OutputAnalyzer run() throws Exception { 287 return runTest("-XX:+PrintGC", 288 "-XX:+PrintGCDetails", 289 "-XX:+UseStringDeduplication", 290 "-XX:+PrintStringDeduplicationStatistics", 291 "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold, 292 InternedTest.class.getName(), 293 "" + DefaultAgeThreshold); 294 } 295 } 296 297 private static class MemoryUsageTest { 298 public static void main(String[] args) { 299 System.out.println("Begin: MemoryUsageTest"); 300 301 final boolean useStringDeduplication = Boolean.parseBoolean(args[0]); 302 final int numberOfStrings = LargeNumberOfStrings; 303 final int numberOfUniqueStrings = 1; 304 305 ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings); 306 forceDeduplication(DefaultAgeThreshold, FullGC); 307 308 if (useStringDeduplication) { 309 verifyStrings(list, numberOfUniqueStrings); 310 } 311 312 System.gc(); 313 System.out.println("Heap Memory Usage: " + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed()); 314 315 System.out.println("End: MemoryUsageTest"); 316 } 317 318 public static OutputAnalyzer run(boolean useStringDeduplication) throws Exception { 319 String[] extraArgs = new String[0]; 320 321 if (useStringDeduplication) { 322 extraArgs = new String[] { 323 "-XX:+UseStringDeduplication", 324 "-XX:+PrintStringDeduplicationStatistics", 325 "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold 326 }; 327 } 328 329 String[] defaultArgs = new String[] { 330 "-XX:+PrintGC", 331 "-XX:+PrintGCDetails", 332 MemoryUsageTest.class.getName(), 333 "" + useStringDeduplication 334 }; 335 336 ArrayList<String> args = new ArrayList<String>(); 337 args.addAll(Arrays.asList(extraArgs)); 338 args.addAll(Arrays.asList(defaultArgs)); 339 340 return runTest(args.toArray(new String[args.size()])); 341 } 342 } 343 344 /* 345 * Tests 346 */ 347 348 private static final int LargeNumberOfStrings = 10000; 349 private static final int SmallNumberOfStrings = 10; 350 351 private static final int MaxAgeThreshold = 15; 352 private static final int DefaultAgeThreshold = 3; 353 private static final int MinAgeThreshold = 1; 354 355 private static final int TooLowAgeThreshold = MinAgeThreshold - 1; 356 private static final int TooHighAgeThreshold = MaxAgeThreshold + 1; 357 358 public static void testYoungGC() throws Exception { 359 // Do young GC to age strings to provoke deduplication 360 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings, 361 DefaultAgeThreshold, 362 YoungGC, 363 "-XX:+PrintGC", 364 "-XX:+PrintStringDeduplicationStatistics"); 365 output.shouldNotContain("Full GC"); 366 output.shouldContain("GC pause (G1 Evacuation Pause) (young)"); 367 output.shouldContain("GC concurrent-string-deduplication"); 368 output.shouldContain("Deduplicated:"); 369 output.shouldHaveExitValue(0); 370 } 371 372 public static void testFullGC() throws Exception { 373 // Do full GC to age strings to provoke deduplication 374 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings, 375 DefaultAgeThreshold, 376 FullGC, 377 "-XX:+PrintGC", 378 "-XX:+PrintStringDeduplicationStatistics"); 379 output.shouldNotContain("GC pause (G1 Evacuation Pause) (young)"); 380 output.shouldContain("Full GC"); 381 output.shouldContain("GC concurrent-string-deduplication"); 382 output.shouldContain("Deduplicated:"); 383 output.shouldHaveExitValue(0); 384 } 385 386 public static void testTableResize() throws Exception { 387 // Test with StringDeduplicationResizeALot 388 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings, 389 DefaultAgeThreshold, 390 YoungGC, 391 "-XX:+PrintGC", 392 "-XX:+PrintStringDeduplicationStatistics", 393 "-XX:+StringDeduplicationResizeALot"); 394 output.shouldContain("GC concurrent-string-deduplication"); 395 output.shouldContain("Deduplicated:"); 396 output.shouldNotContain("Resize Count: 0"); 397 output.shouldHaveExitValue(0); 398 } 399 400 public static void testTableRehash() throws Exception { 401 // Test with StringDeduplicationRehashALot 402 OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings, 403 DefaultAgeThreshold, 404 YoungGC, 405 "-XX:+PrintGC", 406 "-XX:+PrintStringDeduplicationStatistics", 407 "-XX:+StringDeduplicationRehashALot"); 408 output.shouldContain("GC concurrent-string-deduplication"); 409 output.shouldContain("Deduplicated:"); 410 output.shouldNotContain("Rehash Count: 0"); 411 output.shouldNotContain("Hash Seed: 0x0"); 412 output.shouldHaveExitValue(0); 413 } 414 415 public static void testAgeThreshold() throws Exception { 416 OutputAnalyzer output; 417 418 // Test with max age theshold 419 output = DeduplicationTest.run(SmallNumberOfStrings, 420 MaxAgeThreshold, 421 YoungGC, 422 "-XX:+PrintGC", 423 "-XX:+PrintStringDeduplicationStatistics"); 424 output.shouldContain("GC concurrent-string-deduplication"); 425 output.shouldContain("Deduplicated:"); 426 output.shouldHaveExitValue(0); 427 428 // Test with min age theshold 429 output = DeduplicationTest.run(SmallNumberOfStrings, 430 MinAgeThreshold, 431 YoungGC, 432 "-XX:+PrintGC", 433 "-XX:+PrintStringDeduplicationStatistics"); 434 output.shouldContain("GC concurrent-string-deduplication"); 435 output.shouldContain("Deduplicated:"); 436 output.shouldHaveExitValue(0); 437 438 // Test with too low age threshold 439 output = DeduplicationTest.run(SmallNumberOfStrings, 440 TooLowAgeThreshold, 441 YoungGC); 442 output.shouldContain("StringDeduplicationAgeThreshold of " + TooLowAgeThreshold + 443 " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold); 444 output.shouldHaveExitValue(1); 445 446 // Test with too high age threshold 447 output = DeduplicationTest.run(SmallNumberOfStrings, 448 TooHighAgeThreshold, 449 YoungGC); 450 output.shouldContain("StringDeduplicationAgeThreshold of " + TooHighAgeThreshold + 451 " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold); 452 output.shouldHaveExitValue(1); 453 } 454 455 public static void testPrintOptions() throws Exception { 456 OutputAnalyzer output; 457 458 // Test without PrintGC and without PrintStringDeduplicationStatistics 459 output = DeduplicationTest.run(SmallNumberOfStrings, 460 DefaultAgeThreshold, 461 YoungGC); 462 output.shouldNotContain("GC concurrent-string-deduplication"); 463 output.shouldNotContain("Deduplicated:"); 464 output.shouldHaveExitValue(0); 465 466 // Test with PrintGC but without PrintStringDeduplicationStatistics 467 output = DeduplicationTest.run(SmallNumberOfStrings, 468 DefaultAgeThreshold, 469 YoungGC, 470 "-XX:+PrintGC"); 471 output.shouldContain("GC concurrent-string-deduplication"); 472 output.shouldNotContain("Deduplicated:"); 473 output.shouldHaveExitValue(0); 474 } 475 476 public static void testInterned() throws Exception { 477 // Test that interned strings are deduplicated before being interned 478 OutputAnalyzer output = InternedTest.run(); 479 output.shouldHaveExitValue(0); 480 } 481 482 public static void testMemoryUsage() throws Exception { 483 // Test that memory usage is reduced after deduplication 484 OutputAnalyzer output; 485 final String usagePattern = "Heap Memory Usage: (\\d+)"; 486 487 // Run without deduplication 488 output = MemoryUsageTest.run(false); 489 output.shouldHaveExitValue(0); 490 final long memoryUsageWithoutDedup = Long.parseLong(output.firstMatch(usagePattern, 1)); 491 492 // Run with deduplication 493 output = MemoryUsageTest.run(true); 494 output.shouldHaveExitValue(0); 495 final long memoryUsageWithDedup = Long.parseLong(output.firstMatch(usagePattern, 1)); 496 497 // Calculate expected memory usage with deduplication enabled. This calculation does 498 // not take alignment and padding into account, so it's a conservative estimate. 499 final long sizeOfChar = 2; // bytes 500 final long bytesSaved = (LargeNumberOfStrings - 1) * (StringLength * sizeOfChar + unsafe.ARRAY_CHAR_BASE_OFFSET); 501 final long memoryUsageWithDedupExpected = memoryUsageWithoutDedup - bytesSaved; 502 503 System.out.println("Memory usage summary:"); 504 System.out.println(" memoryUsageWithoutDedup: " + memoryUsageWithoutDedup); 505 System.out.println(" memoryUsageWithDedup: " + memoryUsageWithDedup); 506 System.out.println(" memoryUsageWithDedupExpected: " + memoryUsageWithDedupExpected); 507 508 if (memoryUsageWithDedup > memoryUsageWithDedupExpected) { 509 throw new Exception("Unexpected memory usage, memoryUsageWithDedup should less or equal to memoryUsageWithDedupExpected"); 510 } 511 } 512 }