--- old/make/autoconf/hotspot.m4 2018-06-01 22:29:44.433456083 +0200 +++ new/make/autoconf/hotspot.m4 2018-06-01 22:29:44.200446025 +0200 @@ -25,7 +25,7 @@ # All valid JVM features, regardless of platform VALID_JVM_FEATURES="compiler1 compiler2 zero minimal dtrace jvmti jvmci \ - graal vm-structs jni-check services management cmsgc g1gc parallelgc serialgc nmt cds \ + graal vm-structs jni-check services management cmsgc g1gc parallelgc serialgc zgc nmt cds \ static-build link-time-opt aot jfr" # Deprecated JVM features (these are ignored, but with a warning) @@ -328,6 +328,19 @@ fi fi + # Only enable ZGC on Linux x86 + AC_MSG_CHECKING([if zgc should be built]) + if HOTSPOT_CHECK_JVM_FEATURE(zgc); then + if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then + AC_MSG_RESULT([yes]) + else + DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES zgc" + AC_MSG_RESULT([no, platform not supported]) + fi + else + AC_MSG_RESULT([no]) + fi + # Turn on additional features based on other parts of configure if test "x$INCLUDE_DTRACE" = "xtrue"; then JVM_FEATURES="$JVM_FEATURES dtrace" --- old/make/hotspot/lib/JvmFeatures.gmk 2018-06-01 22:29:44.780471061 +0200 +++ new/make/hotspot/lib/JvmFeatures.gmk 2018-06-01 22:29:44.528460183 +0200 @@ -155,6 +155,11 @@ JVM_EXCLUDE_FILES += psMarkSweep.cpp psMarkSweepDecorator.cpp endif +ifneq ($(call check-jvm-feature, zgc), true) + JVM_CFLAGS_FEATURES += -DINCLUDE_ZGC=0 + JVM_EXCLUDE_PATTERNS += gc/z +endif + ifneq ($(call check-jvm-feature, jfr), true) JVM_CFLAGS_FEATURES += -DINCLUDE_JFR=0 JVM_EXCLUDE_PATTERNS += jfr --- old/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp 2018-06-01 22:29:45.170487896 +0200 +++ new/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp 2018-06-01 22:29:44.911476716 +0200 @@ -1346,7 +1346,13 @@ __ decode_heap_oop(dest->as_register()); } #endif - __ verify_oop(dest->as_register()); +#if INCLUDE_ZGC + // Load barrier not yet applied, so a verification here would fail + if (!UseZGC) +#endif + { + __ verify_oop(dest->as_register()); + } } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { #ifdef _LP64 if (UseCompressedClassPointers) { --- old/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp 2018-06-01 22:29:45.611506932 +0200 +++ new/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp 2018-06-01 22:29:45.354495838 +0200 @@ -44,6 +44,9 @@ #ifdef COMPILER2 #include "opto/runtime.hpp" #endif +#if INCLUDE_ZGC +#include "gc/z/zThreadLocalData.hpp" +#endif // Declaration and definition of StubGenerator (no .hpp file). // For a more detailed description of the stub routine structure @@ -1026,6 +1029,15 @@ // make sure object is 'reasonable' __ testptr(rax, rax); __ jcc(Assembler::zero, exit); // if obj is NULL it is OK + +#if INCLUDE_ZGC + if (UseZGC) { + // Check if metadata bits indicate a bad oop + __ testptr(rax, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); + __ jcc(Assembler::notZero, error); + } +#endif + // Check if the oop is in the right area of memory __ movptr(c_rarg2, rax); __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask()); --- old/src/hotspot/cpu/x86/x86.ad 2018-06-01 22:29:46.047525752 +0200 +++ new/src/hotspot/cpu/x86/x86.ad 2018-06-01 22:29:45.789514616 +0200 @@ -1067,6 +1067,138 @@ #endif ); +reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d); +reg_class ymm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h); +reg_class zmm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p); + +reg_class xmm1_reg(XMM1, XMM1b, XMM1c, XMM1d); +reg_class ymm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h); +reg_class zmm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p); + +reg_class xmm2_reg(XMM2, XMM2b, XMM2c, XMM2d); +reg_class ymm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h); +reg_class zmm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p); + +reg_class xmm3_reg(XMM3, XMM3b, XMM3c, XMM3d); +reg_class ymm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h); +reg_class zmm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p); + +reg_class xmm4_reg(XMM4, XMM4b, XMM4c, XMM4d); +reg_class ymm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h); +reg_class zmm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p); + +reg_class xmm5_reg(XMM5, XMM5b, XMM5c, XMM5d); +reg_class ymm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h); +reg_class zmm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p); + +reg_class xmm6_reg(XMM6, XMM6b, XMM6c, XMM6d); +reg_class ymm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h); +reg_class zmm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p); + +reg_class xmm7_reg(XMM7, XMM7b, XMM7c, XMM7d); +reg_class ymm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h); +reg_class zmm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p); + +#ifdef _LP64 + +reg_class xmm8_reg(XMM8, XMM8b, XMM8c, XMM8d); +reg_class ymm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h); +reg_class zmm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p); + +reg_class xmm9_reg(XMM9, XMM9b, XMM9c, XMM9d); +reg_class ymm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h); +reg_class zmm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p); + +reg_class xmm10_reg(XMM10, XMM10b, XMM10c, XMM10d); +reg_class ymm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h); +reg_class zmm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p); + +reg_class xmm11_reg(XMM11, XMM11b, XMM11c, XMM11d); +reg_class ymm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h); +reg_class zmm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p); + +reg_class xmm12_reg(XMM12, XMM12b, XMM12c, XMM12d); +reg_class ymm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h); +reg_class zmm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p); + +reg_class xmm13_reg(XMM13, XMM13b, XMM13c, XMM13d); +reg_class ymm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h); +reg_class zmm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p); + +reg_class xmm14_reg(XMM14, XMM14b, XMM14c, XMM14d); +reg_class ymm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h); +reg_class zmm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p); + +reg_class xmm15_reg(XMM15, XMM15b, XMM15c, XMM15d); +reg_class ymm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h); +reg_class zmm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p); + +reg_class xmm16_reg(XMM16, XMM16b, XMM16c, XMM16d); +reg_class ymm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h); +reg_class zmm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p); + +reg_class xmm17_reg(XMM17, XMM17b, XMM17c, XMM17d); +reg_class ymm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h); +reg_class zmm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p); + +reg_class xmm18_reg(XMM18, XMM18b, XMM18c, XMM18d); +reg_class ymm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h); +reg_class zmm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p); + +reg_class xmm19_reg(XMM19, XMM19b, XMM19c, XMM19d); +reg_class ymm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h); +reg_class zmm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p); + +reg_class xmm20_reg(XMM20, XMM20b, XMM20c, XMM20d); +reg_class ymm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h); +reg_class zmm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p); + +reg_class xmm21_reg(XMM21, XMM21b, XMM21c, XMM21d); +reg_class ymm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h); +reg_class zmm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p); + +reg_class xmm22_reg(XMM22, XMM22b, XMM22c, XMM22d); +reg_class ymm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h); +reg_class zmm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p); + +reg_class xmm23_reg(XMM23, XMM23b, XMM23c, XMM23d); +reg_class ymm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h); +reg_class zmm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p); + +reg_class xmm24_reg(XMM24, XMM24b, XMM24c, XMM24d); +reg_class ymm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h); +reg_class zmm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p); + +reg_class xmm25_reg(XMM25, XMM25b, XMM25c, XMM25d); +reg_class ymm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h); +reg_class zmm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p); + +reg_class xmm26_reg(XMM26, XMM26b, XMM26c, XMM26d); +reg_class ymm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h); +reg_class zmm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p); + +reg_class xmm27_reg(XMM27, XMM27b, XMM27c, XMM27d); +reg_class ymm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h); +reg_class zmm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p); + +reg_class xmm28_reg(XMM28, XMM28b, XMM28c, XMM28d); +reg_class ymm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h); +reg_class zmm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p); + +reg_class xmm29_reg(XMM29, XMM29b, XMM29c, XMM29d); +reg_class ymm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h); +reg_class zmm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p); + +reg_class xmm30_reg(XMM30, XMM30b, XMM30c, XMM30d); +reg_class ymm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h); +reg_class zmm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p); + +reg_class xmm31_reg(XMM31, XMM31b, XMM31c, XMM31d); +reg_class ymm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h); +reg_class zmm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p); + +#endif + %} --- old/src/hotspot/cpu/x86/x86_64.ad 2018-06-01 22:29:46.564548069 +0200 +++ new/src/hotspot/cpu/x86/x86_64.ad 2018-06-01 22:29:46.308537019 +0200 @@ -538,6 +538,12 @@ %} +source_hpp %{ +#if INCLUDE_ZGC +#include "gc/z/zBarrierSetAssembler.hpp" +#endif +%} + //----------SOURCE BLOCK------------------------------------------------------- // This is a block of C++ code which provides values, functions, and // definitions necessary in the rest of the architecture description @@ -4221,6 +4227,135 @@ %} %} +// Operands for bound floating pointer register arguments +operand rxmm0() %{ + constraint(ALLOC_IN_RC(xmm0_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX<= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm1() %{ + constraint(ALLOC_IN_RC(xmm1_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm2() %{ + constraint(ALLOC_IN_RC(xmm2_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm3() %{ + constraint(ALLOC_IN_RC(xmm3_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm4() %{ + constraint(ALLOC_IN_RC(xmm4_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm5() %{ + constraint(ALLOC_IN_RC(xmm5_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm6() %{ + constraint(ALLOC_IN_RC(xmm6_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm7() %{ + constraint(ALLOC_IN_RC(xmm7_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm8() %{ + constraint(ALLOC_IN_RC(xmm8_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm9() %{ + constraint(ALLOC_IN_RC(xmm9_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm10() %{ + constraint(ALLOC_IN_RC(xmm10_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm11() %{ + constraint(ALLOC_IN_RC(xmm11_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm12() %{ + constraint(ALLOC_IN_RC(xmm12_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm13() %{ + constraint(ALLOC_IN_RC(xmm13_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm14() %{ + constraint(ALLOC_IN_RC(xmm14_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm15() %{ + constraint(ALLOC_IN_RC(xmm15_reg)); match(VecX); + predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER); +%} +operand rxmm16() %{ + constraint(ALLOC_IN_RC(xmm16_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm17() %{ + constraint(ALLOC_IN_RC(xmm17_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm18() %{ + constraint(ALLOC_IN_RC(xmm18_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm19() %{ + constraint(ALLOC_IN_RC(xmm19_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm20() %{ + constraint(ALLOC_IN_RC(xmm20_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm21() %{ + constraint(ALLOC_IN_RC(xmm21_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm22() %{ + constraint(ALLOC_IN_RC(xmm22_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm23() %{ + constraint(ALLOC_IN_RC(xmm23_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm24() %{ + constraint(ALLOC_IN_RC(xmm24_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm25() %{ + constraint(ALLOC_IN_RC(xmm25_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm26() %{ + constraint(ALLOC_IN_RC(xmm26_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm27() %{ + constraint(ALLOC_IN_RC(xmm27_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm28() %{ + constraint(ALLOC_IN_RC(xmm28_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm29() %{ + constraint(ALLOC_IN_RC(xmm29_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm30() %{ + constraint(ALLOC_IN_RC(xmm30_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} +operand rxmm31() %{ + constraint(ALLOC_IN_RC(xmm31_reg)); match(VecX); + predicate(UseAVX == 3); format%{%} interface(REG_INTER); +%} //----------OPERAND CLASSES---------------------------------------------------- // Operand Classes are groups of operands that are used as to simplify @@ -11547,6 +11682,16 @@ ins_pipe(ialu_cr_reg_mem); %} +instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero) +%{ + match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero)); + + format %{ "testq $src, $mem" %} + opcode(0x85); + ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem)); + ins_pipe(ialu_cr_reg_mem); +%} + // Manifest a CmpL result in an integer register. Very painful. // This is the test to avoid. instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags) @@ -12340,6 +12485,223 @@ ins_pipe(pipe_jmp); %} +// +// Execute ZGC load barrier (strong) slow path +// + +// When running without XMM regs +instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ + + match(Set dst (LoadBarrierSlowReg mem)); + predicate(MaxVectorSize < 16); + + effect(DEF dst, KILL cr); + + format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} + ins_encode %{ +#if INCLUDE_ZGC + Register d = $dst$$Register; + ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + + assert(d != r12, "Can't be R12!"); + assert(d != r15, "Can't be R15!"); + assert(d != rsp, "Can't be RSP!"); + + __ lea(d, $mem$$Address); + __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); +#else + ShouldNotReachHere(); +#endif + %} + ins_pipe(pipe_slow); +%} + +// For XMM and YMM enabled processors +instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, + rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, + rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, + rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ + + match(Set dst (LoadBarrierSlowReg mem)); + predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16)); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, + KILL x4, KILL x5, KILL x6, KILL x7, + KILL x8, KILL x9, KILL x10, KILL x11, + KILL x12, KILL x13, KILL x14, KILL x15); + + format %{"LoadBarrierSlowRegXmm $dst, $mem" %} + ins_encode %{ +#if INCLUDE_ZGC + Register d = $dst$$Register; + ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + + assert(d != r12, "Can't be R12!"); + assert(d != r15, "Can't be R15!"); + assert(d != rsp, "Can't be RSP!"); + + __ lea(d, $mem$$Address); + __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); +#else + ShouldNotReachHere(); +#endif + %} + ins_pipe(pipe_slow); +%} + +// For ZMM enabled processors +instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, + rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, + rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, + rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15, + rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19, + rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23, + rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27, + rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ + + match(Set dst (LoadBarrierSlowReg mem)); + predicate((UseAVX == 3) && (MaxVectorSize >= 16)); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, + KILL x4, KILL x5, KILL x6, KILL x7, + KILL x8, KILL x9, KILL x10, KILL x11, + KILL x12, KILL x13, KILL x14, KILL x15, + KILL x16, KILL x17, KILL x18, KILL x19, + KILL x20, KILL x21, KILL x22, KILL x23, + KILL x24, KILL x25, KILL x26, KILL x27, + KILL x28, KILL x29, KILL x30, KILL x31); + + format %{"LoadBarrierSlowRegZmm $dst, $mem" %} + ins_encode %{ +#if INCLUDE_ZGC + Register d = $dst$$Register; + ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + + assert(d != r12, "Can't be R12!"); + assert(d != r15, "Can't be R15!"); + assert(d != rsp, "Can't be RSP!"); + + __ lea(d, $mem$$Address); + __ call(RuntimeAddress(bs->load_barrier_slow_stub(d))); +#else + ShouldNotReachHere(); +#endif + %} + ins_pipe(pipe_slow); +%} + +// +// Execute ZGC load barrier (weak) slow path +// + +// When running without XMM regs +instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{ + + match(Set dst (LoadBarrierSlowReg mem)); + predicate(MaxVectorSize < 16); + + effect(DEF dst, KILL cr); + + format %{"LoadBarrierSlowRegNoVec $dst, $mem" %} + ins_encode %{ +#if INCLUDE_ZGC + Register d = $dst$$Register; + ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + + assert(d != r12, "Can't be R12!"); + assert(d != r15, "Can't be R15!"); + assert(d != rsp, "Can't be RSP!"); + + __ lea(d, $mem$$Address); + __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); +#else + ShouldNotReachHere(); +#endif + %} + ins_pipe(pipe_slow); +%} + +// For XMM and YMM enabled processors +instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, + rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, + rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, + rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{ + + match(Set dst (LoadBarrierWeakSlowReg mem)); + predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16)); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, + KILL x4, KILL x5, KILL x6, KILL x7, + KILL x8, KILL x9, KILL x10, KILL x11, + KILL x12, KILL x13, KILL x14, KILL x15); + + format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %} + ins_encode %{ +#if INCLUDE_ZGC + Register d = $dst$$Register; + ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + + assert(d != r12, "Can't be R12!"); + assert(d != r15, "Can't be R15!"); + assert(d != rsp, "Can't be RSP!"); + + __ lea(d,$mem$$Address); + __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); +#else + ShouldNotReachHere(); +#endif + %} + ins_pipe(pipe_slow); +%} + +// For ZMM enabled processors +instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr, + rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3, + rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7, + rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11, + rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15, + rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19, + rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23, + rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27, + rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{ + + match(Set dst (LoadBarrierWeakSlowReg mem)); + predicate((UseAVX == 3) && (MaxVectorSize >= 16)); + + effect(DEF dst, KILL cr, + KILL x0, KILL x1, KILL x2, KILL x3, + KILL x4, KILL x5, KILL x6, KILL x7, + KILL x8, KILL x9, KILL x10, KILL x11, + KILL x12, KILL x13, KILL x14, KILL x15, + KILL x16, KILL x17, KILL x18, KILL x19, + KILL x20, KILL x21, KILL x22, KILL x23, + KILL x24, KILL x25, KILL x26, KILL x27, + KILL x28, KILL x29, KILL x30, KILL x31); + + format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %} + ins_encode %{ +#if INCLUDE_ZGC + Register d = $dst$$Register; + ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler(); + + assert(d != r12, "Can't be R12!"); + assert(d != r15, "Can't be R15!"); + assert(d != rsp, "Can't be RSP!"); + + __ lea(d,$mem$$Address); + __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d))); +#else + ShouldNotReachHere(); +#endif + %} + ins_pipe(pipe_slow); +%} // ============================================================================ // This name is KNOWN by the ADLC and cannot be changed. --- old/src/hotspot/share/adlc/formssel.cpp 2018-06-01 22:29:47.047568918 +0200 +++ new/src/hotspot/share/adlc/formssel.cpp 2018-06-01 22:29:46.796558084 +0200 @@ -2282,6 +2282,9 @@ if (strcmp(name, "RegD") == 0) size = 2; if (strcmp(name, "RegL") == 0) size = 2; if (strcmp(name, "RegN") == 0) size = 1; + if (strcmp(name, "VecX") == 0) size = 4; + if (strcmp(name, "VecY") == 0) size = 8; + if (strcmp(name, "VecZ") == 0) size = 16; if (strcmp(name, "RegP") == 0) size = globalAD->get_preproc_def("_LP64") ? 2 : 1; if (size == 0) { return false; @@ -3509,6 +3512,7 @@ "ClearArray", "GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP", "GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN", + "LoadBarrierSlowReg", "LoadBarrierWeakSlowReg" }; int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*); if( strcmp(_opType,"PrefetchAllocation")==0 ) --- old/src/hotspot/share/classfile/vmSymbols.cpp 2018-06-01 22:29:47.472587264 +0200 +++ new/src/hotspot/share/classfile/vmSymbols.cpp 2018-06-01 22:29:47.215576170 +0200 @@ -756,6 +756,9 @@ #endif // COMPILER1 #ifdef COMPILER2 case vmIntrinsics::_clone: +#if INCLUDE_ZGC + if (UseZGC) return true; +#endif case vmIntrinsics::_copyOf: case vmIntrinsics::_copyOfRange: // These intrinsics use both the objectcopy and the arraycopy --- old/src/hotspot/share/compiler/compilerDirectives.hpp 2018-06-01 22:29:47.854603753 +0200 +++ new/src/hotspot/share/compiler/compilerDirectives.hpp 2018-06-01 22:29:47.594592530 +0200 @@ -66,7 +66,8 @@ cflags(VectorizeDebug, uintx, 0, VectorizeDebug) \ cflags(CloneMapDebug, bool, false, CloneMapDebug) \ cflags(IGVPrintLevel, intx, PrintIdealGraphLevel, IGVPrintLevel) \ - cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) + cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) \ +ZGC_ONLY(cflags(ZOptimizeLoadBarriers, bool, ZOptimizeLoadBarriers, ZOptimizeLoadBarriers)) #else #define compilerdirectives_c2_flags(cflags) #endif --- old/src/hotspot/share/compiler/oopMap.cpp 2018-06-01 22:29:48.223619682 +0200 +++ new/src/hotspot/share/compiler/oopMap.cpp 2018-06-01 22:29:47.966608588 +0200 @@ -380,8 +380,12 @@ continue; } #ifdef ASSERT - if ((((uintptr_t)loc & (sizeof(*loc)-1)) != 0) || - !Universe::heap()->is_in_or_null(*loc)) { + // We can not verify the oop here if we are using ZGC, the oop + // will be bad in case we had a safepoint between a load and a + // load barrier. + if (ZGC_ONLY(!UseZGC &&) + ((((uintptr_t)loc & (sizeof(*loc)-1)) != 0) || + !Universe::heap()->is_in_or_null(*loc))) { tty->print_cr("# Found non oop pointer. Dumping state at failure"); // try to dump out some helpful debugging information trace_codeblob_maps(fr, reg_map); --- old/src/hotspot/share/gc/shared/barrierSetConfig.hpp 2018-06-01 22:29:48.621636862 +0200 +++ new/src/hotspot/share/gc/shared/barrierSetConfig.hpp 2018-06-01 22:29:48.365625811 +0200 @@ -30,7 +30,8 @@ // Do something for each concrete barrier set part of the build. #define FOR_EACH_CONCRETE_BARRIER_SET_DO(f) \ f(CardTableBarrierSet) \ - G1GC_ONLY(f(G1BarrierSet)) + G1GC_ONLY(f(G1BarrierSet)) \ + ZGC_ONLY(f(ZBarrierSet)) #define FOR_EACH_ABSTRACT_BARRIER_SET_DO(f) \ f(ModRef) --- old/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp 2018-06-01 22:29:48.984652531 +0200 +++ new/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp 2018-06-01 22:29:48.726641394 +0200 @@ -26,12 +26,13 @@ #define SHARE_VM_GC_SHARED_BARRIERSETCONFIG_INLINE_HPP #include "gc/shared/barrierSetConfig.hpp" - #include "gc/shared/modRefBarrierSet.inline.hpp" #include "gc/shared/cardTableBarrierSet.inline.hpp" - #if INCLUDE_G1GC -#include "gc/g1/g1BarrierSet.inline.hpp" // G1 support +#include "gc/g1/g1BarrierSet.inline.hpp" +#endif +#if INCLUDE_ZGC +#include "gc/z/zBarrierSet.inline.hpp" #endif #endif // SHARE_VM_GC_SHARED_BARRIERSETCONFIG_INLINE_HPP --- old/src/hotspot/share/gc/shared/collectedHeap.hpp 2018-06-01 22:29:49.361668805 +0200 +++ new/src/hotspot/share/gc/shared/collectedHeap.hpp 2018-06-01 22:29:49.104657711 +0200 @@ -89,6 +89,7 @@ // CMSHeap // G1CollectedHeap // ParallelScavengeHeap +// ZCollectedHeap // class CollectedHeap : public CHeapObj { friend class VMStructs; @@ -196,7 +197,8 @@ Serial, Parallel, CMS, - G1 + G1, + Z }; static inline size_t filler_array_max_size() { --- old/src/hotspot/share/gc/shared/gcCause.cpp 2018-06-01 22:29:49.736684992 +0200 +++ new/src/hotspot/share/gc/shared/gcCause.cpp 2018-06-01 22:29:49.481673984 +0200 @@ -105,6 +105,21 @@ case _dcmd_gc_run: return "Diagnostic Command"; + case _z_timer: + return "Timer"; + + case _z_warmup: + return "Warmup"; + + case _z_allocation_rate: + return "Allocation Rate"; + + case _z_allocation_stall: + return "Allocation Stall"; + + case _z_proactive: + return "Proactive"; + case _last_gc_cause: return "ILLEGAL VALUE - last gc cause - ILLEGAL VALUE"; --- old/src/hotspot/share/gc/shared/gcCause.hpp 2018-06-01 22:29:50.122701654 +0200 +++ new/src/hotspot/share/gc/shared/gcCause.hpp 2018-06-01 22:29:49.863690474 +0200 @@ -78,6 +78,12 @@ _dcmd_gc_run, + _z_timer, + _z_warmup, + _z_allocation_rate, + _z_allocation_stall, + _z_proactive, + _last_gc_cause }; --- old/src/hotspot/share/gc/shared/gcConfig.cpp 2018-06-01 22:29:50.508718316 +0200 +++ new/src/hotspot/share/gc/shared/gcConfig.cpp 2018-06-01 22:29:50.251707222 +0200 @@ -40,6 +40,9 @@ #if INCLUDE_SERIALGC #include "gc/serial/serialArguments.hpp" #endif +#if INCLUDE_ZGC +#include "gc/z/zArguments.hpp" +#endif struct SupportedGC { bool& _flag; @@ -55,6 +58,7 @@ G1GC_ONLY(static G1Arguments g1Arguments;) PARALLELGC_ONLY(static ParallelArguments parallelArguments;) SERIALGC_ONLY(static SerialArguments serialArguments;) + ZGC_ONLY(static ZArguments zArguments;) // Table of supported GCs, for translating between command // line flag, CollectedHeap::Name and GCArguments instance. @@ -64,6 +68,7 @@ PARALLELGC_ONLY_ARG(SupportedGC(UseParallelGC, CollectedHeap::Parallel, parallelArguments, "parallel gc")) PARALLELGC_ONLY_ARG(SupportedGC(UseParallelOldGC, CollectedHeap::Parallel, parallelArguments, "parallel gc")) SERIALGC_ONLY_ARG(SupportedGC(UseSerialGC, CollectedHeap::Serial, serialArguments, "serial gc")) + ZGC_ONLY_ARG(SupportedGC(UseZGC, CollectedHeap::Z, zArguments, "z gc")) }; #define FOR_EACH_SUPPORTED_GC(var) \ @@ -92,6 +97,7 @@ NOT_PARALLELGC(UNSUPPORTED_OPTION(UseParallelGC);) NOT_PARALLELGC(UNSUPPORTED_OPTION(UseParallelOldGC)); NOT_SERIALGC( UNSUPPORTED_OPTION(UseSerialGC);) + NOT_ZGC( UNSUPPORTED_OPTION(UseZGC);) } bool GCConfig::is_no_gc_selected() { --- old/src/hotspot/share/gc/shared/gcConfiguration.cpp 2018-06-01 22:29:50.856733338 +0200 +++ new/src/hotspot/share/gc/shared/gcConfiguration.cpp 2018-06-01 22:29:50.600722287 +0200 @@ -43,6 +43,10 @@ return ParNew; } + if (UseZGC) { + return NA; + } + return DefNew; } @@ -59,6 +63,10 @@ return ParallelOld; } + if (UseZGC) { + return Z; + } + return SerialOld; } --- old/src/hotspot/share/gc/shared/gcName.hpp 2018-06-01 22:29:51.237749784 +0200 +++ new/src/hotspot/share/gc/shared/gcName.hpp 2018-06-01 22:29:50.978738604 +0200 @@ -38,6 +38,8 @@ ConcurrentMarkSweep, G1Old, G1Full, + Z, + NA, GCNameEndSentinel }; @@ -55,6 +57,8 @@ case ConcurrentMarkSweep: return "ConcurrentMarkSweep"; case G1Old: return "G1Old"; case G1Full: return "G1Full"; + case Z: return "Z"; + case NA: return "N/A"; default: ShouldNotReachHere(); return NULL; } } --- old/src/hotspot/share/gc/shared/gcThreadLocalData.hpp 2018-06-01 22:29:51.627766619 +0200 +++ new/src/hotspot/share/gc/shared/gcThreadLocalData.hpp 2018-06-01 22:29:51.368755439 +0200 @@ -40,6 +40,6 @@ // should consider placing frequently accessed fields first in // T, so that field offsets relative to Thread are small, which // often allows for a more compact instruction encoding. -typedef uint64_t GCThreadLocalData[14]; // 112 bytes +typedef uint64_t GCThreadLocalData[18]; // 144 bytes #endif // SHARE_GC_SHARED_GCTHREADLOCALDATA_HPP --- old/src/hotspot/share/gc/shared/gc_globals.hpp 2018-06-01 22:29:52.019783540 +0200 +++ new/src/hotspot/share/gc/shared/gc_globals.hpp 2018-06-01 22:29:51.759772317 +0200 @@ -38,6 +38,9 @@ #if INCLUDE_SERIALGC #include "gc/serial/serial_globals.hpp" #endif +#if INCLUDE_ZGC +#include "gc/z/zFlags.hpp" +#endif #define GC_FLAGS(develop, \ develop_pd, \ @@ -118,6 +121,22 @@ constraint, \ writeable)) \ \ + ZGC_ONLY(GC_Z_FLAGS( \ + develop, \ + develop_pd, \ + product, \ + product_pd, \ + diagnostic, \ + diagnostic_pd, \ + experimental, \ + notproduct, \ + manageable, \ + product_rw, \ + lp64_product, \ + range, \ + constraint, \ + writeable)) \ + \ /* gc */ \ \ product(bool, UseConcMarkSweepGC, false, \ @@ -135,6 +154,9 @@ product(bool, UseParallelOldGC, false, \ "Use the Parallel Old garbage collector") \ \ + experimental(bool, UseZGC, false, \ + "Use the Z garbage collector") \ + \ product(uint, ParallelGCThreads, 0, \ "Number of parallel threads parallel gc will use") \ constraint(ParallelGCThreadsConstraintFunc,AfterErgo) \ --- old/src/hotspot/share/gc/shared/specialized_oop_closures.hpp 2018-06-01 22:29:52.431801324 +0200 +++ new/src/hotspot/share/gc/shared/specialized_oop_closures.hpp 2018-06-01 22:29:52.176790317 +0200 @@ -35,6 +35,9 @@ #if INCLUDE_SERIALGC #include "gc/serial/serial_specialized_oop_closures.hpp" #endif +#if INCLUDE_ZGC +#include "gc/z/zOopClosures.specialized.hpp" +#endif // The following OopClosure types get specialized versions of // "oop_oop_iterate" that invoke the closures' do_oop methods @@ -67,7 +70,8 @@ SERIALGC_ONLY(SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_MS(f)) \ CMSGC_ONLY(SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_CMS(f)) \ G1GC_ONLY(SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(f)) \ - G1GC_ONLY(SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1FULL(f)) + G1GC_ONLY(SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1FULL(f)) \ + ZGC_ONLY(SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_Z(f)) // We separate these out, because sometime the general one has // a different definition from the specialized ones, and sometimes it --- old/src/hotspot/share/gc/shared/vmStructs_gc.hpp 2018-06-01 22:29:52.776816216 +0200 +++ new/src/hotspot/share/gc/shared/vmStructs_gc.hpp 2018-06-01 22:29:52.523805295 +0200 @@ -47,6 +47,9 @@ #include "gc/serial/defNewGeneration.hpp" #include "gc/serial/vmStructs_serial.hpp" #endif +#if INCLUDE_ZGC +#include "gc/z/vmStructs_z.hpp" +#endif #define VM_STRUCTS_GC(nonstatic_field, \ volatile_nonstatic_field, \ @@ -64,6 +67,10 @@ SERIALGC_ONLY(VM_STRUCTS_SERIALGC(nonstatic_field, \ volatile_nonstatic_field, \ static_field)) \ + ZGC_ONLY(VM_STRUCTS_ZGC(nonstatic_field, \ + volatile_nonstatic_field, \ + static_field)) \ + \ /**********************************************************************************/ \ /* Generation and Space hierarchies */ \ /**********************************************************************************/ \ @@ -162,6 +169,10 @@ SERIALGC_ONLY(VM_TYPES_SERIALGC(declare_type, \ declare_toplevel_type, \ declare_integer_type)) \ + ZGC_ONLY(VM_TYPES_ZGC(declare_type, \ + declare_toplevel_type, \ + declare_integer_type)) \ + \ /******************************************/ \ /* Generation and space hierarchies */ \ /* (needed for run-time type information) */ \ @@ -231,6 +242,8 @@ declare_constant_with_value)) \ SERIALGC_ONLY(VM_INT_CONSTANTS_SERIALGC(declare_constant, \ declare_constant_with_value)) \ + ZGC_ONLY(VM_INT_CONSTANTS_ZGC(declare_constant, \ + declare_constant_with_value)) \ \ /********************************************/ \ /* Generation and Space Hierarchy Constants */ \ @@ -274,5 +287,7 @@ declare_constant(Generation::LogOfGenGrain) \ declare_constant(Generation::GenGrain) \ +#define VM_LONG_CONSTANTS_GC(declare_constant) \ + ZGC_ONLY(VM_LONG_CONSTANTS_ZGC(declare_constant)) #endif // SHARE_GC_SHARED_VMSTRUCTS_GC_HPP --- old/src/hotspot/share/jfr/metadata/metadata.xml 2018-06-01 22:29:53.148832274 +0200 +++ new/src/hotspot/share/jfr/metadata/metadata.xml 2018-06-01 22:29:52.892821224 +0200 @@ -885,6 +885,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1089,4 +1122,4 @@ - \ No newline at end of file + --- old/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.cpp 2018-06-01 22:29:53.547849497 +0200 +++ new/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.cpp 2018-06-01 22:29:53.295838620 +0200 @@ -59,6 +59,9 @@ #include "gc/g1/g1HeapRegionTraceType.hpp" #include "gc/g1/g1YCTypes.hpp" #endif +#if INCLUDE_ZGC +#include "gc/z/zStat.hpp" +#endif // implementation for the static registration function exposed in the api bool JfrSerializer::register_serializer(JfrTypeId id, bool require_safepoint, bool permit_cache, JfrSerializer* cs) { @@ -346,3 +349,27 @@ writer.write(thread_group_id); JfrThreadGroup::serialize(&writer, thread_group_id); } + +void ZStatCounterTypeConstant::serialize(JfrCheckpointWriter& writer) { +#if INCLUDE_ZGC + writer.write_count(ZStatCounter::count()); + for (ZStatCounter* counter = ZStatCounter::first(); counter != NULL; counter = counter->next()) { + writer.write_key(counter->id()); + writer.write(counter->name()); + } +#else + writer.write_count(0); +#endif +} + +void ZStatSamplerTypeConstant::serialize(JfrCheckpointWriter& writer) { +#if INCLUDE_ZGC + writer.write_count(ZStatSampler::count()); + for (ZStatSampler* sampler = ZStatSampler::first(); sampler != NULL; sampler = sampler->next()) { + writer.write_key(sampler->id()); + writer.write(sampler->name()); + } +#else + writer.write_count(0); +#endif +} --- old/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.hpp 2018-06-01 22:29:53.944866634 +0200 +++ new/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.hpp 2018-06-01 22:29:53.682855325 +0200 @@ -135,4 +135,14 @@ void serialize(JfrCheckpointWriter& writer); }; +class ZStatCounterTypeConstant : public JfrSerializer { + public: + void serialize(JfrCheckpointWriter& writer); +}; + +class ZStatSamplerTypeConstant : public JfrSerializer { + public: + void serialize(JfrCheckpointWriter& writer); +}; + #endif // SHARE_VM_JFR_CHECKPOINT_CONSTANT_JFRCONSTANT_HPP --- old/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp 2018-06-01 22:29:54.332883383 +0200 +++ new/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp 2018-06-01 22:29:54.075872289 +0200 @@ -195,7 +195,7 @@ bool JfrTypeManager::initialize() { // register non-safepointing type serialization - for (size_t i = 0; i < 16; ++i) { + for (size_t i = 0; i < 18; ++i) { switch (i) { case 0: register_serializer(TYPE_FLAGVALUEORIGIN, false, true, new FlagValueOriginConstant()); break; case 1: register_serializer(TYPE_INFLATECAUSE, false, true, new MonitorInflateCauseConstant()); break; @@ -213,6 +213,8 @@ case 13: register_serializer(TYPE_CODEBLOBTYPE, false, true, new CodeBlobTypeConstant()); break; case 14: register_serializer(TYPE_VMOPERATIONTYPE, false, true, new VMOperationTypeConstant()); break; case 15: register_serializer(TYPE_THREADSTATE, false, true, new ThreadStateConstant()); break; + case 16: register_serializer(TYPE_ZSTATCOUNTERTYPE, false, true, new ZStatCounterTypeConstant()); break; + case 17: register_serializer(TYPE_ZSTATSAMPLERTYPE, false, true, new ZStatSamplerTypeConstant()); break; default: guarantee(false, "invariant"); } --- old/src/hotspot/share/logging/logPrefix.hpp 2018-06-01 22:29:54.723900261 +0200 +++ new/src/hotspot/share/logging/logPrefix.hpp 2018-06-01 22:29:54.465889124 +0200 @@ -62,9 +62,11 @@ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, humongous)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ihop)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, liveness)) \ + LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, load)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, marking)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, metaspace)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, mmu)) \ + LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, nmethod)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, phases)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, phases, ref)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, phases, start)) \ @@ -75,6 +77,7 @@ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, remset, tracking)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ref)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ref, start)) \ + LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, reloc)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, start)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, stringtable)) \ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, sweep)) \ --- old/src/hotspot/share/logging/logTag.hpp 2018-06-01 22:29:55.095916318 +0200 +++ new/src/hotspot/share/logging/logTag.hpp 2018-06-01 22:29:54.839905268 +0200 @@ -62,6 +62,7 @@ LOG_TAG(datacreation) \ LOG_TAG(decoder) \ LOG_TAG(defaultmethods) \ + LOG_TAG(director) \ LOG_TAG(dump) \ LOG_TAG(ergo) \ LOG_TAG(exceptions) \ @@ -105,6 +106,7 @@ LOG_TAG(obsolete) \ LOG_TAG(oopmap) \ LOG_TAG(oopstorage) \ + LOG_TAG(oops) \ LOG_TAG(os) \ LOG_TAG(pagesize) \ LOG_TAG(patch) \ @@ -120,6 +122,7 @@ LOG_TAG(redefine) \ LOG_TAG(refine) \ LOG_TAG(region) \ + LOG_TAG(reloc) \ LOG_TAG(remset) \ LOG_TAG(purge) \ LOG_TAG(resolve) \ --- old/src/hotspot/share/memory/metaspace.hpp 2018-06-01 22:29:55.484933110 +0200 +++ new/src/hotspot/share/memory/metaspace.hpp 2018-06-01 22:29:55.225921930 +0200 @@ -229,6 +229,7 @@ // Manages the metaspace portion belonging to a class loader class ClassLoaderMetaspace : public CHeapObj { friend class CollectedHeap; // For expand_and_allocate() + friend class ZCollectedHeap; // For expand_and_allocate() friend class Metaspace; friend class MetaspaceUtils; friend class metaspace::PrintCLDMetaspaceInfoClosure; --- old/src/hotspot/share/oops/instanceRefKlass.inline.hpp 2018-06-01 22:29:55.880950204 +0200 +++ new/src/hotspot/share/oops/instanceRefKlass.inline.hpp 2018-06-01 22:29:55.625939196 +0200 @@ -57,9 +57,13 @@ bool InstanceRefKlass::try_discover(oop obj, ReferenceType type, OopClosureType* closure) { ReferenceDiscoverer* rd = closure->ref_discoverer(); if (rd != NULL) { - T referent_oop = RawAccess<>::oop_load((T*)java_lang_ref_Reference::referent_addr_raw(obj)); - if (!CompressedOops::is_null(referent_oop)) { - oop referent = CompressedOops::decode_not_null(referent_oop); + oop referent; + if (type == REF_PHANTOM) { + referent = HeapAccess::oop_load(java_lang_ref_Reference::referent_addr_raw(obj)); + } else { + referent = HeapAccess::oop_load(java_lang_ref_Reference::referent_addr_raw(obj)); + } + if (referent != NULL) { if (!referent->is_gc_marked()) { // Only try to discover if not yet marked. return rd->discover_reference(obj, type); @@ -179,9 +183,9 @@ log_develop_trace(gc, ref)("InstanceRefKlass %s for obj " PTR_FORMAT, s, p2i(obj)); log_develop_trace(gc, ref)(" referent_addr/* " PTR_FORMAT " / " PTR_FORMAT, - p2i(referent_addr), p2i(referent_addr ? RawAccess<>::oop_load(referent_addr) : (oop)NULL)); + p2i(referent_addr), p2i((oop)HeapAccess::oop_load_at(obj, java_lang_ref_Reference::referent_offset))); log_develop_trace(gc, ref)(" discovered_addr/* " PTR_FORMAT " / " PTR_FORMAT, - p2i(discovered_addr), p2i(discovered_addr ? RawAccess<>::oop_load(discovered_addr) : (oop)NULL)); + p2i(discovered_addr), p2i((oop)HeapAccess::oop_load(discovered_addr))); } #endif --- old/src/hotspot/share/opto/classes.cpp 2018-06-01 22:29:56.257966477 +0200 +++ new/src/hotspot/share/opto/classes.cpp 2018-06-01 22:29:55.998955297 +0200 @@ -47,11 +47,16 @@ #include "opto/rootnode.hpp" #include "opto/subnode.hpp" #include "opto/vectornode.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ZGC +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif // ---------------------------------------------------------------------------- // Build a table of virtual functions to map from Nodes to dense integer // opcode names. int Node::Opcode() const { return Op_Node; } #define macro(x) int x##Node::Opcode() const { return Op_##x; } +#define optionalmacro(x) #include "classes.hpp" #undef macro --- old/src/hotspot/share/opto/classes.hpp 2018-06-01 22:29:56.631982621 +0200 +++ new/src/hotspot/share/opto/classes.hpp 2018-06-01 22:29:56.383971916 +0200 @@ -22,6 +22,8 @@ * */ +#include "utilities/macros.hpp" + // The giant table of Node classes. // One entry per class, sorted by class name. @@ -186,6 +188,14 @@ macro(LoadN) macro(LoadRange) macro(LoadS) +#if INCLUDE_ZGC +#define zgcmacro(x) macro(x) +#else +#define zgcmacro(x) optionalmacro(x) +#endif +zgcmacro(LoadBarrier) +zgcmacro(LoadBarrierSlowReg) +zgcmacro(LoadBarrierWeakSlowReg) macro(Lock) macro(Loop) macro(LoopLimit) --- old/src/hotspot/share/opto/compile.cpp 2018-06-01 22:29:56.976997514 +0200 +++ new/src/hotspot/share/opto/compile.cpp 2018-06-01 22:29:56.722986549 +0200 @@ -75,9 +75,13 @@ #include "runtime/timer.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" +#include "utilities/macros.hpp" #if INCLUDE_G1GC #include "gc/g1/g1ThreadLocalData.hpp" #endif // INCLUDE_G1GC +#if INCLUDE_ZGC +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif // -------------------- Compile::mach_constant_base_node ----------------------- @@ -2163,6 +2167,11 @@ #endif +#ifdef ASSERT + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + bs->verify_gc_barriers(true); +#endif + ResourceMark rm; int loop_opts_cnt; @@ -2335,6 +2344,12 @@ } } +#if INCLUDE_ZGC + if (UseZGC) { + ZBarrierSetC2::find_dominating_barriers(igvn); + } +#endif + if (failing()) return; // Ensure that major progress is now clear @@ -2358,6 +2373,8 @@ bs->verify_gc_barriers(false); #endif + print_method(PHASE_BEFORE_MACRO_EXPANSION, 2); + { TracePhase tp("macroExpand", &timers[_t_macroExpand]); PhaseMacroExpand mex(igvn); @@ -2890,6 +2907,10 @@ case Op_LoadL_unaligned: case Op_LoadPLocked: case Op_LoadP: +#if INCLUDE_ZGC + case Op_LoadBarrierSlowReg: + case Op_LoadBarrierWeakSlowReg: +#endif case Op_LoadN: case Op_LoadRange: case Op_LoadS: { --- old/src/hotspot/share/opto/compile.hpp 2018-06-01 22:29:57.395015557 +0200 +++ new/src/hotspot/share/opto/compile.hpp 2018-06-01 22:29:57.139004506 +0200 @@ -54,6 +54,7 @@ class ConnectionGraph; class InlineTree; class Int_Array; +class LoadBarrierNode; class Matcher; class MachConstantNode; class MachConstantBaseNode; @@ -359,9 +360,6 @@ const char* _stub_name; // Name of stub or adapter being compiled, or NULL address _stub_entry_point; // Compile code entry for generated stub, or NULL - // For GC - void* _barrier_set_state; - // Control of this compilation. int _num_loop_opts; // Number of iterations for doing loop optimiztions int _max_inline_size; // Max inline size for this compilation @@ -410,6 +408,7 @@ // Compilation environment. Arena _comp_arena; // Arena with lifetime equivalent to Compile + void* _barrier_set_state; // Potential GC barrier state for Compile ciEnv* _env; // CI interface DirectiveSet* _directive; // Compiler directive CompileLog* _log; // from CompilerThread --- old/src/hotspot/share/opto/escape.cpp 2018-06-01 22:29:57.790032608 +0200 +++ new/src/hotspot/share/opto/escape.cpp 2018-06-01 22:29:57.533021514 +0200 @@ -38,9 +38,13 @@ #include "opto/phaseX.hpp" #include "opto/movenode.hpp" #include "opto/rootnode.hpp" +#include "utilities/macros.hpp" #if INCLUDE_G1GC #include "gc/g1/g1ThreadLocalData.hpp" #endif // INCLUDE_G1GC +#if INCLUDE_ZGC +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif ConnectionGraph::ConnectionGraph(Compile * C, PhaseIterGVN *igvn) : _nodes(C->comp_arena(), C->unique(), C->unique(), NULL), @@ -449,6 +453,10 @@ break; } case Op_LoadP: +#if INCLUDE_ZGC + case Op_LoadBarrierSlowReg: + case Op_LoadBarrierWeakSlowReg: +#endif case Op_LoadN: case Op_LoadPLocked: { add_objload_to_connection_graph(n, delayed_worklist); @@ -483,6 +491,13 @@ add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist); } +#if INCLUDE_ZGC + else if (UseZGC) { + if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist); + } + } +#endif break; } case Op_Rethrow: // Exception object escapes @@ -651,6 +666,10 @@ break; } case Op_LoadP: +#if INCLUDE_ZGC + case Op_LoadBarrierSlowReg: + case Op_LoadBarrierWeakSlowReg: +#endif case Op_LoadN: case Op_LoadPLocked: { // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because @@ -690,6 +709,14 @@ add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), NULL); break; } +#if INCLUDE_ZGC + else if (UseZGC) { + if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL); + break; + } + } +#endif ELSE_FAIL("Op_Proj"); } case Op_Rethrow: // Exception object escapes @@ -3163,7 +3190,8 @@ op == Op_CastP2X || op == Op_StoreCM || op == Op_FastLock || op == Op_AryEq || op == Op_StrComp || op == Op_HasNegatives || op == Op_StrCompressedCopy || op == Op_StrInflatedCopy || - op == Op_StrEquals || op == Op_StrIndexOf || op == Op_StrIndexOfChar)) { + op == Op_StrEquals || op == Op_StrIndexOf || op == Op_StrIndexOfChar || + BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(use))) { n->dump(); use->dump(); assert(false, "EA: missing allocation reference path"); --- old/src/hotspot/share/opto/idealKit.cpp 2018-06-01 22:29:58.204050478 +0200 +++ new/src/hotspot/share/opto/idealKit.cpp 2018-06-01 22:29:57.946039341 +0200 @@ -488,13 +488,13 @@ //----------------------------- make_call ---------------------------- // Trivial runtime call -void IdealKit::make_leaf_call(const TypeFunc *slow_call_type, - address slow_call, - const char *leaf_name, - Node* parm0, - Node* parm1, - Node* parm2, - Node* parm3) { +Node* IdealKit::make_leaf_call(const TypeFunc *slow_call_type, + address slow_call, + const char *leaf_name, + Node* parm0, + Node* parm1, + Node* parm2, + Node* parm3) { // We only handle taking in RawMem and modifying RawMem const TypePtr* adr_type = TypeRawPtr::BOTTOM; @@ -532,6 +532,12 @@ assert(C->alias_type(call->adr_type()) == C->alias_type(adr_type), "call node must be constructed correctly"); + Node* res = NULL; + if (slow_call_type->range()->cnt() > TypeFunc::Parms) { + assert(slow_call_type->range()->cnt() == TypeFunc::Parms+1, "only one return value"); + res = transform(new ProjNode(call, TypeFunc::Parms)); + } + return res; } void IdealKit::make_leaf_call_no_fp(const TypeFunc *slow_call_type, --- old/src/hotspot/share/opto/idealKit.hpp 2018-06-01 22:29:58.572066363 +0200 +++ new/src/hotspot/share/opto/idealKit.hpp 2018-06-01 22:29:58.313055183 +0200 @@ -242,13 +242,13 @@ int adr_idx); // Trivial call - void make_leaf_call(const TypeFunc *slow_call_type, - address slow_call, - const char *leaf_name, - Node* parm0, - Node* parm1 = NULL, - Node* parm2 = NULL, - Node* parm3 = NULL); + Node* make_leaf_call(const TypeFunc *slow_call_type, + address slow_call, + const char *leaf_name, + Node* parm0, + Node* parm1 = NULL, + Node* parm2 = NULL, + Node* parm3 = NULL); void make_leaf_call_no_fp(const TypeFunc *slow_call_type, address slow_call, --- old/src/hotspot/share/opto/lcm.cpp 2018-06-01 22:29:58.929081774 +0200 +++ new/src/hotspot/share/opto/lcm.cpp 2018-06-01 22:29:58.674070766 +0200 @@ -169,6 +169,8 @@ case Op_LoadI: case Op_LoadL: case Op_LoadP: + case Op_LoadBarrierSlowReg: + case Op_LoadBarrierWeakSlowReg: case Op_LoadN: case Op_LoadS: case Op_LoadKlass: --- old/src/hotspot/share/opto/loopnode.cpp 2018-06-01 22:29:59.292097443 +0200 +++ new/src/hotspot/share/opto/loopnode.cpp 2018-06-01 22:29:59.040086565 +0200 @@ -25,6 +25,8 @@ #include "precompiled.hpp" #include "ci/ciMethodData.hpp" #include "compiler/compileLog.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/c2/barrierSetC2.hpp" #include "libadt/vectset.hpp" #include "memory/allocation.inline.hpp" #include "memory/resourceArea.hpp" @@ -213,7 +215,8 @@ if (nb_ctl_proj > 1) { break; } - assert(parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call(), "unexpected node"); + assert(parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() || + BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl), "unexpected node"); assert(idom(ctl) == parent_ctl, "strange"); next = idom(parent_ctl); } @@ -2635,7 +2638,7 @@ //----------------------------build_and_optimize------------------------------- // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. -void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts) { +void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, bool last_round) { ResourceMark rm; int old_progress = C->major_progress(); @@ -2882,8 +2885,11 @@ // that require basic-block info (like cloning through Phi's) if( SplitIfBlocks && do_split_ifs ) { visited.Clear(); - split_if_with_blocks( visited, nstack ); + split_if_with_blocks( visited, nstack, last_round ); NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); ); + if (last_round) { + C->set_major_progress(); + } } if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) { @@ -4136,6 +4142,8 @@ case Op_LoadL: case Op_LoadS: case Op_LoadP: + case Op_LoadBarrierSlowReg: + case Op_LoadBarrierWeakSlowReg: case Op_LoadN: case Op_LoadRange: case Op_LoadD_unaligned: --- old/src/hotspot/share/opto/loopnode.hpp 2018-06-01 22:29:59.711115529 +0200 +++ new/src/hotspot/share/opto/loopnode.hpp 2018-06-01 22:29:59.456104522 +0200 @@ -912,7 +912,7 @@ } // build the loop tree and perform any requested optimizations - void build_and_optimize(bool do_split_if, bool skip_loop_opts); + void build_and_optimize(bool do_split_if, bool skip_loop_opts, bool last_round = false); // Dominators for the sea of nodes void Dominators(); @@ -922,13 +922,13 @@ Node *dom_lca_internal( Node *n1, Node *n2 ) const; // Compute the Ideal Node to Loop mapping - PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs, bool skip_loop_opts = false) : + PhaseIdealLoop( PhaseIterGVN &igvn, bool do_split_ifs, bool skip_loop_opts = false, bool last_round = false) : PhaseTransform(Ideal_Loop), _igvn(igvn), _dom_lca_tags(arena()), // Thread::resource_area _verify_me(NULL), _verify_only(false) { - build_and_optimize(do_split_ifs, skip_loop_opts); + build_and_optimize(do_split_ifs, skip_loop_opts, last_round); } // Verify that verify_me made the same decisions as a fresh run. @@ -1227,9 +1227,9 @@ // Check for aggressive application of 'split-if' optimization, // using basic block level info. - void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack ); + void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack, bool last_round ); Node *split_if_with_blocks_pre ( Node *n ); - void split_if_with_blocks_post( Node *n ); + void split_if_with_blocks_post( Node *n, bool last_round ); Node *has_local_phi_input( Node *n ); // Mark an IfNode as being dominated by a prior test, // without actually altering the CFG (and hence IDOM info). --- old/src/hotspot/share/opto/loopopts.cpp 2018-06-01 22:30:00.113132882 +0200 +++ new/src/hotspot/share/opto/loopopts.cpp 2018-06-01 22:29:59.855121745 +0200 @@ -40,6 +40,10 @@ #include "opto/opaquenode.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ZGC +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif //============================================================================= //------------------------------split_thru_phi--------------------------------- @@ -1126,11 +1130,11 @@ // Do the real work in a non-recursive function. CFG hackery wants to be // in the post-order, so it can dirty the I-DOM info and not use the dirtied // info. -void PhaseIdealLoop::split_if_with_blocks_post(Node *n) { +void PhaseIdealLoop::split_if_with_blocks_post(Node *n, bool last_round) { // Cloning Cmp through Phi's involves the split-if transform. // FastLock is not used by an If - if (n->is_Cmp() && !n->is_FastLock()) { + if (n->is_Cmp() && !n->is_FastLock() && !last_round) { Node *n_ctrl = get_ctrl(n); // Determine if the Node has inputs from some local Phi. // Returns the block to clone thru. @@ -1377,12 +1381,18 @@ get_loop(get_ctrl(n)) == get_loop(get_ctrl(n->in(1))) ) { _igvn.replace_node( n, n->in(1) ); } + +#if INCLUDE_ZGC + if (UseZGC) { + ZBarrierSetC2::loop_optimize_gc_barrier(this, n, last_round); + } +#endif } //------------------------------split_if_with_blocks--------------------------- // Check for aggressive application of 'split-if' optimization, // using basic block level info. -void PhaseIdealLoop::split_if_with_blocks( VectorSet &visited, Node_Stack &nstack ) { +void PhaseIdealLoop::split_if_with_blocks(VectorSet &visited, Node_Stack &nstack, bool last_round) { Node *n = C->root(); visited.set(n->_idx); // first, mark node as visited // Do pre-visit work for root @@ -1407,7 +1417,7 @@ // All of n's children have been processed, complete post-processing. if (cnt != 0 && !n->is_Con()) { assert(has_node(n), "no dead nodes"); - split_if_with_blocks_post( n ); + split_if_with_blocks_post( n, last_round ); } if (nstack.is_empty()) { // Finished all nodes on stack. --- old/src/hotspot/share/opto/macro.cpp 2018-06-01 22:30:00.555151961 +0200 +++ new/src/hotspot/share/opto/macro.cpp 2018-06-01 22:30:00.296140781 +0200 @@ -2574,7 +2574,9 @@ assert(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2 || - n->Opcode() == Op_Opaque3, "unknown node type in macro list"); + n->Opcode() == Op_Opaque3 || + BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n), + "unknown node type in macro list"); } assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count"); progress = progress || success; @@ -2656,7 +2658,7 @@ while (macro_idx >= 0) { Node * n = C->macro_node(macro_idx); assert(n->is_macro(), "only macro nodes expected here"); - if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) { + if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { // node is unreachable, so don't try to expand it C->remove_macro_node(n); } else if (n->is_ArrayCopy()){ @@ -2674,7 +2676,7 @@ int macro_count = C->macro_count(); Node * n = C->macro_node(macro_count-1); assert(n->is_macro(), "only macro nodes expected here"); - if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) { + if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { // node is unreachable, so don't try to expand it C->remove_macro_node(n); continue; --- old/src/hotspot/share/opto/matcher.cpp 2018-06-01 22:30:00.946168839 +0200 +++ new/src/hotspot/share/opto/matcher.cpp 2018-06-01 22:30:00.687157659 +0200 @@ -41,6 +41,9 @@ #include "runtime/os.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/align.hpp" +#if INCLUDE_ZGC +#include "gc/z/zBarrierSetRuntime.hpp" +#endif // INCLUDE_ZGC OptoReg::Name OptoReg::c_frame_pointer; @@ -2062,6 +2065,7 @@ mstack.set_state(Post_Visit); set_visited(n); // Flag as visited now bool mem_op = false; + int mem_addr_idx = MemNode::Address; switch( nop ) { // Handle some opcodes special case Op_Phi: // Treat Phis as shared roots @@ -2150,6 +2154,17 @@ case Op_SafePoint: mem_op = true; break; +#if INCLUDE_ZGC + case Op_CallLeaf: + if (UseZGC) { + if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() || + n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) { + mem_op = true; + mem_addr_idx = TypeFunc::Parms+1; + } + break; + } +#endif default: if( n->is_Store() ) { // Do match stores, despite no ideal reg @@ -2199,7 +2214,7 @@ #endif // Clone addressing expressions as they are "free" in memory access instructions - if (mem_op && i == MemNode::Address && mop == Op_AddP && + if (mem_op && i == mem_addr_idx && mop == Op_AddP && // When there are other uses besides address expressions // put it on stack and mark as shared. !is_visited(m)) { --- old/src/hotspot/share/opto/memnode.cpp 2018-06-01 22:30:01.391188048 +0200 +++ new/src/hotspot/share/opto/memnode.cpp 2018-06-01 22:30:01.131176825 +0200 @@ -44,7 +44,11 @@ #include "opto/regmask.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" +#include "utilities/macros.hpp" #include "utilities/vmError.hpp" +#if INCLUDE_ZGC +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif // Portions of code courtesy of Clifford Click @@ -891,6 +895,14 @@ // a load node that reads from the source array so we may be able to // optimize out the ArrayCopy node later. Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const { +#if INCLUDE_ZGC + if (UseZGC) { + if (bottom_type()->make_oopptr() != NULL) { + return NULL; + } + } +#endif + Node* ld_adr = in(MemNode::Address); intptr_t ld_off = 0; AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off); @@ -1574,7 +1586,7 @@ // Is there a dominating load that loads the same value? Leave // anything that is not a load of a field/array element (like // barriers etc.) alone - if (in(0) != NULL && adr_type() != TypeRawPtr::BOTTOM && can_reshape) { + if (in(0) != NULL && !adr_type()->isa_rawptr() && can_reshape) { for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { Node *use = mem->fast_out(i); if (use != this && @@ -2968,6 +2980,16 @@ return NULL; } +#if INCLUDE_ZGC + if (UseZGC) { + if (req() == (Precedent+1) && in(MemBarNode::Precedent)->in(0) != NULL && in(MemBarNode::Precedent)->in(0)->is_LoadBarrier()) { + Node* load_node = in(MemBarNode::Precedent)->in(0)->in(LoadBarrierNode::Oop); + set_req(MemBarNode::Precedent, load_node); + return this; + } + } +#endif + bool progress = false; // Eliminate volatile MemBars for scalar replaced objects. if (can_reshape && req() == (Precedent+1)) { --- old/src/hotspot/share/opto/node.cpp 2018-06-01 22:30:01.853207991 +0200 +++ new/src/hotspot/share/opto/node.cpp 2018-06-01 22:30:01.593196768 +0200 @@ -1130,7 +1130,7 @@ if (this->is_Store()) { // Condition for back-to-back stores folding. return n->Opcode() == op && n->in(MemNode::Memory) == this; - } else if (this->is_Load() || this->is_DecodeN()) { + } else if (this->is_Load() || this->is_DecodeN() || this->is_Phi()) { // Condition for removing an unused LoadNode or DecodeNNode from the MemBarAcquire precedence input return n->Opcode() == Op_MemBarAcquire; } else if (op == Op_AddL) { --- old/src/hotspot/share/opto/node.hpp 2018-06-01 22:30:02.270225991 +0200 +++ new/src/hotspot/share/opto/node.hpp 2018-06-01 22:30:02.009214725 +0200 @@ -80,6 +80,9 @@ class JumpNode; class JumpProjNode; class LoadNode; +class LoadBarrierNode; +class LoadBarrierSlowRegNode; +class LoadBarrierWeakSlowRegNode; class LoadStoreNode; class LockNode; class LoopNode; @@ -634,6 +637,7 @@ DEFINE_CLASS_ID(MemBar, Multi, 3) DEFINE_CLASS_ID(Initialize, MemBar, 0) DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) + DEFINE_CLASS_ID(LoadBarrier, Multi, 4) DEFINE_CLASS_ID(Mach, Node, 1) DEFINE_CLASS_ID(MachReturn, Mach, 0) @@ -680,6 +684,8 @@ DEFINE_CLASS_ID(Mem, Node, 4) DEFINE_CLASS_ID(Load, Mem, 0) DEFINE_CLASS_ID(LoadVector, Load, 0) + DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1) + DEFINE_CLASS_ID(LoadBarrierWeakSlowReg, Load, 2) DEFINE_CLASS_ID(Store, Mem, 1) DEFINE_CLASS_ID(StoreVector, Store, 0) DEFINE_CLASS_ID(LoadStore, Mem, 2) @@ -819,6 +825,9 @@ DEFINE_CLASS_QUERY(JumpProj) DEFINE_CLASS_QUERY(Load) DEFINE_CLASS_QUERY(LoadStore) + DEFINE_CLASS_QUERY(LoadBarrier) + DEFINE_CLASS_QUERY(LoadBarrierSlowReg) + DEFINE_CLASS_QUERY(LoadBarrierWeakSlowReg) DEFINE_CLASS_QUERY(Lock) DEFINE_CLASS_QUERY(Loop) DEFINE_CLASS_QUERY(Mach) @@ -1717,6 +1726,7 @@ virtual const Type* Value(PhaseGVN* phase) const; virtual const Type *bottom_type() const; virtual uint ideal_reg() const; + #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; virtual void dump_compact_spec(outputStream *st) const; --- old/src/hotspot/share/opto/opcodes.cpp 2018-06-01 22:30:02.680243689 +0200 +++ new/src/hotspot/share/opto/opcodes.cpp 2018-06-01 22:30:02.427232768 +0200 @@ -28,6 +28,7 @@ // Build a table of class names as strings. Used both for debugging printouts // and in the ADL machine descriptions. #define macro(x) #x, +#define optionalmacro(x) macro(x) const char *NodeClassNames[] = { "Node", "Set", --- old/src/hotspot/share/opto/opcodes.hpp 2018-06-01 22:30:03.041259272 +0200 +++ new/src/hotspot/share/opto/opcodes.hpp 2018-06-01 22:30:02.783248135 +0200 @@ -27,6 +27,7 @@ // Build a big enum of class names to give them dense integer indices #define macro(x) Op_##x, +#define optionalmacro(x) macro(x) enum Opcodes { Op_Node = 0, macro(Set) // Instruction selection match rule @@ -47,6 +48,7 @@ _last_opcode }; #undef macro +#undef optionalmacro // Table of names, indexed by Opcode extern const char *NodeClassNames[]; --- old/src/hotspot/share/opto/phasetype.hpp 2018-06-01 22:30:03.419275589 +0200 +++ new/src/hotspot/share/opto/phasetype.hpp 2018-06-01 22:30:03.161264452 +0200 @@ -52,6 +52,7 @@ PHASE_MATCHING, PHASE_INCREMENTAL_INLINE, PHASE_INCREMENTAL_BOXING_INLINE, + PHASE_BEFORE_MACRO_EXPANSION, PHASE_END, PHASE_FAILURE, @@ -88,6 +89,7 @@ case PHASE_MATCHING: return "After matching"; case PHASE_INCREMENTAL_INLINE: return "Incremental Inline"; case PHASE_INCREMENTAL_BOXING_INLINE: return "Incremental Boxing Inline"; + case PHASE_BEFORE_MACRO_EXPANSION: return "Before macro expansion"; case PHASE_END: return "End"; case PHASE_FAILURE: return "Failure"; default: --- old/src/hotspot/share/opto/vectornode.cpp 2018-06-01 22:30:03.806292294 +0200 +++ new/src/hotspot/share/opto/vectornode.cpp 2018-06-01 22:30:03.548281157 +0200 @@ -252,6 +252,8 @@ case Op_LoadI: case Op_LoadL: case Op_LoadF: case Op_LoadD: case Op_LoadP: case Op_LoadN: + case Op_LoadBarrierSlowReg: + case Op_LoadBarrierWeakSlowReg: *start = 0; *end = 0; // no vector operands break; --- old/src/hotspot/share/prims/jvmtiTagMap.cpp 2018-06-01 22:30:04.199309258 +0200 +++ new/src/hotspot/share/prims/jvmtiTagMap.cpp 2018-06-01 22:30:03.942298165 +0200 @@ -178,6 +178,8 @@ // hash a given key (oop) with the specified size static unsigned int hash(oop key, int size) { + ZGC_ONLY(assert(ZAddressMetadataShift >= sizeof(unsigned int) * BitsPerByte, "cast removes the metadata bits");) + // shift right to get better distribution (as these bits will be zero // with aligned addresses) unsigned int addr = (unsigned int)(cast_from_oop(key)); --- old/src/hotspot/share/prims/whitebox.cpp 2018-06-01 22:30:04.610327000 +0200 +++ new/src/hotspot/share/prims/whitebox.cpp 2018-06-01 22:30:04.355315992 +0200 @@ -347,7 +347,12 @@ ParallelScavengeHeap* psh = ParallelScavengeHeap::heap(); return !psh->is_in_young(p); } -#endif // INCLUDE_PARALLELGC +#endif +#if INCLUDE_ZGC + if (UseZGC) { + return Universe::heap()->is_in(p); + } +#endif GenCollectedHeap* gch = GenCollectedHeap::heap(); return !gch->is_in_young(p); WB_END --- old/src/hotspot/share/runtime/jniHandles.cpp 2018-06-01 22:30:05.013344395 +0200 +++ new/src/hotspot/share/runtime/jniHandles.cpp 2018-06-01 22:30:04.752333129 +0200 @@ -327,7 +327,13 @@ VerifyJNIHandles verify_handle; oops_do(&verify_handle); - weak_oops_do(&verify_handle); +#if INCLUDE_ZGC + // In ZGC, JNI weaks are handled concurrently, so they can't be verified here + if (!UseZGC) +#endif + { + weak_oops_do(&verify_handle); + } } // This method is implemented here to avoid circular includes between --- old/src/hotspot/share/runtime/stackValue.cpp 2018-06-01 22:30:05.410361532 +0200 +++ new/src/hotspot/share/runtime/stackValue.cpp 2018-06-01 22:30:05.150350309 +0200 @@ -29,6 +29,9 @@ #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/stackValue.hpp" +#if INCLUDE_ZGC +#include "gc/z/zBarrier.inline.hpp" +#endif StackValue* StackValue::create_stack_value(const frame* fr, const RegisterMap* reg_map, ScopeValue* sv) { if (sv->is_location()) { @@ -119,6 +122,13 @@ val = (oop)NULL; } #endif +#if INCLUDE_ZGC + // Deoptimization must make sure all oop have passed load barrier + if (UseZGC) { + val = ZBarrier::load_barrier_on_oop_field_preloaded((oop*)value_addr, val); + } +#endif + Handle h(Thread::current(), val); // Wrap a handle around the oop return new StackValue(h); } --- old/src/hotspot/share/runtime/vmStructs.cpp 2018-06-01 22:30:05.806378626 +0200 +++ new/src/hotspot/share/runtime/vmStructs.cpp 2018-06-01 22:30:05.547367446 +0200 @@ -1278,6 +1278,7 @@ declare_integer_type(intptr_t) \ declare_unsigned_integer_type(uintx) \ declare_unsigned_integer_type(uintptr_t) \ + declare_unsigned_integer_type(uint8_t) \ declare_unsigned_integer_type(uint32_t) \ declare_unsigned_integer_type(uint64_t) \ \ @@ -2610,6 +2611,12 @@ #define VM_LONG_CONSTANTS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant) \ \ + /****************/ \ + /* GC constants */ \ + /****************/ \ + \ + VM_LONG_CONSTANTS_GC(declare_constant) \ + \ /*********************/ \ /* MarkOop constants */ \ /*********************/ \ --- old/src/hotspot/share/runtime/vm_operations.hpp 2018-06-01 22:30:06.261398267 +0200 +++ new/src/hotspot/share/runtime/vm_operations.hpp 2018-06-01 22:30:06.003387130 +0200 @@ -69,6 +69,7 @@ template(CMS_Final_Remark) \ template(G1CollectForAllocation) \ template(G1CollectFull) \ + template(ZOperation) \ template(HandshakeOneThread) \ template(HandshakeAllThreads) \ template(HandshakeFallback) \ --- old/src/hotspot/share/utilities/macros.hpp 2018-06-01 22:30:06.649415015 +0200 +++ new/src/hotspot/share/utilities/macros.hpp 2018-06-01 22:30:06.392403921 +0200 @@ -203,7 +203,25 @@ #define NOT_SERIALGC_RETURN_(code) { return code; } #endif // INCLUDE_SERIALGC -#if INCLUDE_CMSGC || INCLUDE_G1GC || INCLUDE_PARALLELGC +#ifndef INCLUDE_ZGC +#define INCLUDE_ZGC 1 +#endif // INCLUDE_ZGC + +#if INCLUDE_ZGC +#define ZGC_ONLY(x) x +#define ZGC_ONLY_ARG(arg) arg, +#define NOT_ZGC(x) +#define NOT_ZGC_RETURN /* next token must be ; */ +#define NOT_ZGC_RETURN_(code) /* next token must be ; */ +#else +#define ZGC_ONLY(x) +#define ZGC_ONLY_ARG(arg) +#define NOT_ZGC(x) x +#define NOT_ZGC_RETURN {} +#define NOT_ZGC_RETURN_(code) { return code; } +#endif // INCLUDE_ZGC + +#if INCLUDE_CMSGC || INCLUDE_G1GC || INCLUDE_PARALLELGC || INCLUDE_ZGC #define INCLUDE_NOT_ONLY_SERIALGC 1 #else #define INCLUDE_NOT_ONLY_SERIALGC 0 --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/CollectedHeap.java 2018-06-01 22:30:07.020431030 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/CollectedHeap.java 2018-06-01 22:30:06.763419936 +0200 @@ -75,6 +75,14 @@ public abstract CollectedHeapName kind(); + public String oopAddressDescription(OopHandle handle) { + return handle.toString(); + } + + public OopHandle oop_load_at(OopHandle handle, long offset) { + return handle.getOopHandleAt(offset); + } + public void print() { printOn(System.out); } public void printOn(PrintStream tty) { MemRegion mr = reservedRegion(); --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/CollectedHeapName.java 2018-06-01 22:30:07.404447605 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shared/CollectedHeapName.java 2018-06-01 22:30:07.147436512 +0200 @@ -35,6 +35,7 @@ public static final CollectedHeapName PARALLEL = new CollectedHeapName("Parallel"); public static final CollectedHeapName CMS = new CollectedHeapName("CMS"); public static final CollectedHeapName G1 = new CollectedHeapName("G1"); + public static final CollectedHeapName Z = new CollectedHeapName("Z"); public String toString() { return name; --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java 2018-06-01 22:30:07.793464397 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java 2018-06-01 22:30:07.535453260 +0200 @@ -24,17 +24,26 @@ package sun.jvm.hotspot.memory; -import java.io.*; -import java.util.*; -import sun.jvm.hotspot.debugger.*; +import java.io.PrintStream; +import java.util.Observable; +import java.util.Observer; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.debugger.OopHandle; import sun.jvm.hotspot.gc.cms.CMSHeap; -import sun.jvm.hotspot.gc.serial.SerialHeap; -import sun.jvm.hotspot.gc.shared.*; import sun.jvm.hotspot.gc.g1.G1CollectedHeap; -import sun.jvm.hotspot.gc.parallel.*; -import sun.jvm.hotspot.oops.*; -import sun.jvm.hotspot.types.*; -import sun.jvm.hotspot.runtime.*; +import sun.jvm.hotspot.gc.parallel.ParallelScavengeHeap; +import sun.jvm.hotspot.gc.serial.SerialHeap; +import sun.jvm.hotspot.gc.shared.CollectedHeap; +import sun.jvm.hotspot.gc.z.ZCollectedHeap; +import sun.jvm.hotspot.oops.Oop; +import sun.jvm.hotspot.runtime.BasicType; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VirtualConstructor; +import sun.jvm.hotspot.types.AddressField; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; public class Universe { @@ -72,16 +81,33 @@ }); } + private static boolean typeExists(TypeDataBase db, String type) { + try { + db.lookupType(type); + } catch (RuntimeException e) { + return false; + } + return true; + } + + private static void addHeapTypeIfInDB(TypeDataBase db, Class heapClass) { + String heapName = heapClass.getSimpleName(); + if (typeExists(db, heapName)) { + heapConstructor.addMapping(heapName, heapClass); + } + } + private static synchronized void initialize(TypeDataBase db) { Type type = db.lookupType("Universe"); collectedHeapField = type.getAddressField("_collectedHeap"); heapConstructor = new VirtualConstructor(db); - heapConstructor.addMapping("CMSHeap", CMSHeap.class); - heapConstructor.addMapping("SerialHeap", SerialHeap.class); - heapConstructor.addMapping("ParallelScavengeHeap", ParallelScavengeHeap.class); - heapConstructor.addMapping("G1CollectedHeap", G1CollectedHeap.class); + addHeapTypeIfInDB(db, CMSHeap.class); + addHeapTypeIfInDB(db, SerialHeap.class); + addHeapTypeIfInDB(db, ParallelScavengeHeap.class); + addHeapTypeIfInDB(db, G1CollectedHeap.class); + addHeapTypeIfInDB(db, ZCollectedHeap.class); mainThreadGroupField = type.getOopField("_main_thread_group"); systemThreadGroupField = type.getOopField("_system_thread_group"); --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/Oop.java 2018-06-01 22:30:08.179481059 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/Oop.java 2018-06-01 22:30:07.920469879 +0200 @@ -164,7 +164,7 @@ tty.print("null"); } else { obj.printValueOn(tty); - tty.print(" @ " + obj.getHandle()); + tty.print(" @ " + VM.getVM().getUniverse().heap().oopAddressDescription(obj.getHandle())); } } --- old/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/OopField.java 2018-06-01 22:30:08.570497937 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/OopField.java 2018-06-01 22:30:08.314486886 +0200 @@ -54,7 +54,8 @@ if (!isVMField() && !obj.isInstance() && !obj.isArray()) { throw new InternalError(obj.toString()); } - return obj.getHandle().getOopHandleAt(getOffset()); + + return VM.getVM().getUniverse().heap().oop_load_at(obj.getHandle(), getOffset()); } public Oop getValue(VMObject obj) { --- old/src/jdk.jfr/share/conf/jfr/default.jfc 2018-06-01 22:30:08.923513175 +0200 +++ new/src/jdk.jfr/share/conf/jfr/default.jfc 2018-06-01 22:30:08.672502340 +0200 @@ -614,6 +614,25 @@ true + + true + 10 ms + + + + true + 0 ms + + + + true + 10 ms + + + + true + 10 ms + --- old/src/jdk.jfr/share/conf/jfr/profile.jfc 2018-06-01 22:30:09.318530225 +0200 +++ new/src/jdk.jfr/share/conf/jfr/profile.jfc 2018-06-01 22:30:09.060519088 +0200 @@ -614,6 +614,25 @@ true + + true + 10 ms + + + + true + 0 ms + + + + 10 ms + true + + + + true + 10 ms + --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp 2018-06-01 22:30:09.462536441 +0200 @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" +#include "gc/z/zBarrierSetRuntime.hpp" +#include "runtime/stubCodeGenerator.hpp" +#include "utilities/macros.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/z/c1/zBarrierSetC1.hpp" +#endif // COMPILER1 + +#undef __ +#define __ masm-> + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) __ block_comment(str) +#endif + +static void call_vm(MacroAssembler* masm, + address entry_point, + Register arg0, + Register arg1) { + // Setup arguments + if (arg1 == c_rarg0) { + if (arg0 == c_rarg1) { + __ xchgptr(c_rarg1, c_rarg0); + } else { + __ movptr(c_rarg1, arg1); + __ movptr(c_rarg0, arg0); + } + } else { + if (arg0 != c_rarg0) { + __ movptr(c_rarg0, arg0); + } + if (arg1 != c_rarg1) { + __ movptr(c_rarg1, arg1); + } + } + + // Call VM + __ MacroAssembler::call_VM_leaf_base(entry_point, 2); +} + +void ZBarrierSetAssembler::load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register dst, + Address src, + Register tmp1, + Register tmp_thread) { + if (!ZBarrierSet::barrier_needed(decorators, type)) { + // Barrier not needed + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + return; + } + + BLOCK_COMMENT("ZBarrierSetAssembler::load_at {"); + + // Allocate scratch register + Register scratch = tmp1; + if (tmp1 == noreg) { + scratch = r12; + __ push(scratch); + } + + assert_different_registers(dst, scratch); + + Label done; + + // + // Fast Path + // + + // Load address + __ lea(scratch, src); + + // Load oop at address + __ movptr(dst, Address(scratch, 0)); + + // Test address bad mask + __ testptr(dst, address_bad_mask_from_thread(r15_thread)); + __ jcc(Assembler::zero, done); + + // + // Slow path + // + + // Save registers + __ push(rax); + __ push(rcx); + __ push(rdx); + __ push(rdi); + __ push(rsi); + __ push(r8); + __ push(r9); + __ push(r10); + __ push(r11); + + // We may end up here from generate_native_wrapper, then the method may have + // floats as arguments, and we must spill them before calling the VM runtime + // leaf. From the interpreter all floats are passed on the stack. + assert(Argument::n_float_register_parameters_j == 8, "Assumption"); + const int xmm_size = wordSize * 2; + const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j; + __ subptr(rsp, xmm_spill_size); + __ movdqu(Address(rsp, xmm_size * 7), xmm7); + __ movdqu(Address(rsp, xmm_size * 6), xmm6); + __ movdqu(Address(rsp, xmm_size * 5), xmm5); + __ movdqu(Address(rsp, xmm_size * 4), xmm4); + __ movdqu(Address(rsp, xmm_size * 3), xmm3); + __ movdqu(Address(rsp, xmm_size * 2), xmm2); + __ movdqu(Address(rsp, xmm_size * 1), xmm1); + __ movdqu(Address(rsp, xmm_size * 0), xmm0); + + // Call VM + call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, scratch); + + // Restore registers + __ movdqu(xmm0, Address(rsp, xmm_size * 0)); + __ movdqu(xmm1, Address(rsp, xmm_size * 1)); + __ movdqu(xmm2, Address(rsp, xmm_size * 2)); + __ movdqu(xmm3, Address(rsp, xmm_size * 3)); + __ movdqu(xmm4, Address(rsp, xmm_size * 4)); + __ movdqu(xmm5, Address(rsp, xmm_size * 5)); + __ movdqu(xmm6, Address(rsp, xmm_size * 6)); + __ movdqu(xmm7, Address(rsp, xmm_size * 7)); + __ addptr(rsp, xmm_spill_size); + + __ pop(r11); + __ pop(r10); + __ pop(r9); + __ pop(r8); + __ pop(rsi); + __ pop(rdi); + __ pop(rdx); + __ pop(rcx); + + if (dst == rax) { + __ addptr(rsp, wordSize); + } else { + __ movptr(dst, rax); + __ pop(rax); + } + + __ bind(done); + + // Restore scratch register + if (tmp1 == noreg) { + __ pop(scratch); + } + + BLOCK_COMMENT("} ZBarrierSetAssembler::load_at"); +} + +#ifdef ASSERT + +void ZBarrierSetAssembler::store_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Address dst, + Register src, + Register tmp1, + Register tmp2) { + BLOCK_COMMENT("ZBarrierSetAssembler::store_at {"); + + // Verify oop store + if (type == T_OBJECT || type == T_ARRAY) { + // Note that src could be noreg, which means we + // are storing null and can skip verification. + if (src != noreg) { + Label done; + __ testptr(src, address_bad_mask_from_thread(r15_thread)); + __ jcc(Assembler::zero, done); + __ stop("Verify oop store failed"); + __ should_not_reach_here(); + __ bind(done); + } + } + + // Store value + BarrierSetAssembler::store_at(masm, decorators, type, dst, src, tmp1, tmp2); + + BLOCK_COMMENT("} ZBarrierSetAssembler::store_at"); +} + +#endif // ASSERT + +void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register src, + Register dst, + Register count) { + if (!ZBarrierSet::barrier_needed(decorators, type)) { + // Barrier not needed + return; + } + + BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {"); + + // Save registers + __ pusha(); + + // Call VM + call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), src, count); + + // Restore registers + __ popa(); + + BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue"); +} + +void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, + Register jni_env, + Register obj, + Register tmp, + Label& slowpath) { + BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {"); + + // Resolve jobject + BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); + + // Test address bad mask + __ testptr(obj, address_bad_mask_from_jni_env(jni_env)); + __ jcc(Assembler::notZero, slowpath); + + BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); +} + +#ifdef COMPILER1 + +#undef __ +#define __ ce->masm()-> + +void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce, + LIR_Opr ref) const { + __ testptr(ref->as_register(), address_bad_mask_from_thread(r15_thread)); +} + +void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, + ZLoadBarrierStubC1* stub) const { + // Stub entry + __ bind(*stub->entry()); + + Register ref = stub->ref()->as_register(); + Register ref_addr = noreg; + + if (stub->ref_addr()->is_register()) { + // Address already in register + ref_addr = stub->ref_addr()->as_pointer_register(); + } else { + // Load address into tmp register + ce->leal(stub->ref_addr(), stub->tmp(), stub->patch_code(), stub->patch_info()); + ref_addr = stub->tmp()->as_pointer_register(); + } + + assert_different_registers(ref, ref_addr, noreg); + + // Save rax unless it is the result register + if (ref != rax) { + __ push(rax); + } + + // Setup arguments and call runtime stub + __ subptr(rsp, 2 * BytesPerWord); + ce->store_parameter(ref_addr, 1); + ce->store_parameter(ref, 0); + __ call(RuntimeAddress(stub->runtime_stub())); + __ addptr(rsp, 2 * BytesPerWord); + + // Verify result + __ verify_oop(rax, "Bad oop"); + + // Restore rax unless it is the result register + if (ref != rax) { + __ movptr(ref, rax); + __ pop(rax); + } + + // Stub exit + __ jmp(*stub->continuation()); +} + +#undef __ +#define __ sasm-> + +void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, + DecoratorSet decorators) const { + // Enter and save registers + __ enter(); + __ save_live_registers_no_oop_map(true /* save_fpu_registers */); + + // Setup arguments + __ load_parameter(1, c_rarg1); + __ load_parameter(0, c_rarg0); + + // Call VM + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); + + // Restore registers and return + __ restore_live_registers_except_rax(true /* restore_fpu_registers */); + __ leave(); + __ ret(0); +} + +#endif // COMPILER1 + +#undef __ +#define __ cgen->assembler()-> + +// Generates a register specific stub for calling +// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +// +// The raddr register serves as both input and output for this stub. When the stub is +// called the raddr register contains the object field address (oop*) where the bad oop +// was loaded from, which caused the slow path to be taken. On return from the stub the +// raddr register contains the good/healed oop returned from +// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { + // Don't generate stub for invalid registers + if (raddr == rsp || raddr == r12 || raddr == r15) { + return NULL; + } + + // Create stub name + char name[64]; + const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; + os::snprintf(name, sizeof(name), "load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); + + __ align(CodeEntryAlignment); + StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); + address start = __ pc(); + + // Save live registers + if (raddr != rax) { + __ push(rax); + } + if (raddr != rcx) { + __ push(rcx); + } + if (raddr != rdx) { + __ push(rdx); + } + if (raddr != rsi) { + __ push(rsi); + } + if (raddr != rdi) { + __ push(rdi); + } + if (raddr != r8) { + __ push(r8); + } + if (raddr != r9) { + __ push(r9); + } + if (raddr != r10) { + __ push(r10); + } + if (raddr != r11) { + __ push(r11); + } + + // Setup arguments + if (c_rarg1 != raddr) { + __ movq(c_rarg1, raddr); + } + __ movq(c_rarg0, Address(raddr, 0)); + + // Call barrier function + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); + + // Move result returned in rax to raddr, if needed + if (raddr != rax) { + __ movq(raddr, rax); + } + + // Restore saved registers + if (raddr != r11) { + __ pop(r11); + } + if (raddr != r10) { + __ pop(r10); + } + if (raddr != r9) { + __ pop(r9); + } + if (raddr != r8) { + __ pop(r8); + } + if (raddr != rdi) { + __ pop(rdi); + } + if (raddr != rsi) { + __ pop(rsi); + } + if (raddr != rdx) { + __ pop(rdx); + } + if (raddr != rcx) { + __ pop(rcx); + } + if (raddr != rax) { + __ pop(rax); + } + + __ ret(0); + + return start; +} + +#undef __ + +void ZBarrierSetAssembler::barrier_stubs_init() { + // Load barrier stubs + int stub_code_size = 256 * 16; // Rough estimate of code size + + ResourceMark rm; + BufferBlob* bb = BufferBlob::create("zgc_load_barrier_stubs", stub_code_size); + CodeBuffer buf(bb); + StubCodeGenerator cgen(&buf); + + Register rr = as_Register(0); + for (int i = 0; i < RegisterImpl::number_of_registers; i++) { + _load_barrier_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_STRONG_OOP_REF); + _load_barrier_weak_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_WEAK_OOP_REF); + rr = rr->successor(); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp 2018-06-01 22:30:09.859553578 +0200 @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP +#define CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP + +#ifdef COMPILER1 +class LIR_Assembler; +class LIR_OprDesc; +typedef LIR_OprDesc* LIR_Opr; +class StubAssembler; +class ZLoadBarrierStubC1; +#endif // COMPILER1 + +class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { + address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; + address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; + +public: + ZBarrierSetAssembler() : + _load_barrier_slow_stub(), + _load_barrier_weak_slow_stub() {} + + address load_barrier_slow_stub(Register reg) { return _load_barrier_slow_stub[reg->encoding()]; } + address load_barrier_weak_slow_stub(Register reg) { return _load_barrier_weak_slow_stub[reg->encoding()]; } + + virtual void load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register dst, + Address src, + Register tmp1, + Register tmp_thread); + +#ifdef ASSERT + virtual void store_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Address dst, + Register src, + Register tmp1, + Register tmp2); +#endif // ASSERT + + virtual void arraycopy_prologue(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register src, + Register dst, + Register count); + + virtual void try_resolve_jobject_in_native(MacroAssembler* masm, + Register jni_env, + Register obj, + Register tmp, + Label& slowpath); + +#ifdef COMPILER1 + void generate_c1_load_barrier_test(LIR_Assembler* ce, + LIR_Opr ref) const; + + void generate_c1_load_barrier_stub(LIR_Assembler* ce, + ZLoadBarrierStubC1* stub) const; + + void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, + DecoratorSet decorators) const; +#endif // COMPILER1 + + virtual void barrier_stubs_init(); +}; + +#endif // CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zAddress_linux_x86.inline.hpp 2018-06-01 22:30:10.257570758 +0200 @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_X86_ZADDRESS_LINUX_X86_INLINE_HPP +#define OS_CPU_LINUX_X86_ZADDRESS_LINUX_X86_INLINE_HPP + +inline uintptr_t ZAddress::address(uintptr_t value) { + return value; +} + +#endif // OS_CPU_LINUX_X86_ZADDRESS_LINUX_X86_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.cpp 2018-06-01 22:30:10.636587118 +0200 @@ -0,0 +1,361 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zArray.inline.hpp" +#include "gc/z/zBackingFile_linux_x86.hpp" +#include "gc/z/zBackingPath_linux_x86.hpp" +#include "gc/z/zErrno.hpp" +#include "gc/z/zFlags.hpp" +#include "gc/z/zLargePages.inline.hpp" +#include "logging/log.hpp" +#include "runtime/init.hpp" +#include "runtime/os.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +#include +#include +#include +#include +#include +#include + +// Filesystem names +#define ZFILESYSTEM_TMPFS "tmpfs" +#define ZFILESYSTEM_HUGETLBFS "hugetlbfs" + +// Sysfs file for transparent huge page on tmpfs +#define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled" + +// Default mount points +#define ZMOUNTPOINT_TMPFS "/dev/shm" +#define ZMOUNTPOINT_HUGETLBFS "/hugepages" + +// Java heap filename +#define ZFILENAME_HEAP "java_heap" + +// Support for building on older Linux systems +#ifndef __NR_memfd_create +#define __NR_memfd_create 319 +#endif +#ifndef MFD_CLOEXEC +#define MFD_CLOEXEC 0x0001U +#endif +#ifndef MFD_HUGETLB +#define MFD_HUGETLB 0x0004U +#endif +#ifndef O_CLOEXEC +#define O_CLOEXEC 02000000 +#endif +#ifndef O_TMPFILE +#define O_TMPFILE (020000000 | O_DIRECTORY) +#endif + +// Filesystem types, see statfs(2) +#ifndef TMPFS_MAGIC +#define TMPFS_MAGIC 0x01021994 +#endif +#ifndef HUGETLBFS_MAGIC +#define HUGETLBFS_MAGIC 0x958458f6 +#endif + +static int z_memfd_create(const char *name, unsigned int flags) { + return syscall(__NR_memfd_create, name, flags); +} + +ZBackingFile::ZBackingFile() : + _fd(-1), + _filesystem(0), + _initialized(false) { + + // Create backing file + _fd = create_fd(ZFILENAME_HEAP); + if (_fd == -1) { + return; + } + + // Get filesystem type + struct statfs statfs_buf; + if (fstatfs(_fd, &statfs_buf) == -1) { + ZErrno err; + log_error(gc, init)("Failed to determine filesystem type for backing file (%s)", err.to_string()); + return; + } + _filesystem = statfs_buf.f_type; + + // Make sure we're on a supported filesystem + if (!is_tmpfs() && !is_hugetlbfs()) { + log_error(gc, init)("Backing file must be located on a %s or a %s filesystem", ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS); + return; + } + + // Make sure the filesystem type matches requested large page type + if (ZLargePages::is_transparent() && !is_tmpfs()) { + log_error(gc, init)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem", ZFILESYSTEM_TMPFS); + return; + } + + if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) { + log_error(gc, init)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", ZFILESYSTEM_TMPFS); + return; + } + + if (ZLargePages::is_explicit() && !is_hugetlbfs()) { + log_error(gc, init)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled when using a %s filesystem", ZFILESYSTEM_HUGETLBFS); + return; + } + + if (!ZLargePages::is_explicit() && is_hugetlbfs()) { + log_error(gc, init)("-XX:+UseLargePages must be enabled when using a %s filesystem", ZFILESYSTEM_HUGETLBFS); + return; + } + + // Successfully initialized + _initialized = true; +} + +int ZBackingFile::create_mem_fd(const char* name) const { + // Create file name + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : ""); + + // Create file + const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0; + const int fd = z_memfd_create(filename, MFD_CLOEXEC | extra_flags); + if (fd == -1) { + ZErrno err; + log_debug(gc, init)("Failed to create memfd file (%s)", + ((UseLargePages && err == EINVAL) ? "Hugepages not supported" : err.to_string())); + return -1; + } + + log_debug(gc, init)("Heap backed by file /memfd:%s", filename); + + return fd; +} + +int ZBackingFile::create_file_fd(const char* name) const { + const char* const filesystem = ZLargePages::is_explicit() ? ZFILESYSTEM_HUGETLBFS : ZFILESYSTEM_TMPFS; + const char* const mountpoint = ZLargePages::is_explicit() ? ZMOUNTPOINT_HUGETLBFS : ZMOUNTPOINT_TMPFS; + + // Find mountpoint + ZBackingPath path(filesystem, mountpoint); + if (path.get() == NULL) { + log_error(gc, init)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem); + return -1; + } + + // Try to create an anonymous file using the O_TMPFILE flag. Note that this + // flag requires kernel >= 3.11. If this fails we fall back to open/unlink. + const int fd_anon = open(path.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); + if (fd_anon == -1) { + ZErrno err; + log_debug(gc, init)("Failed to create anonymouns file in %s (%s)", path.get(), + (err == EINVAL ? "Not supported" : err.to_string())); + } else { + // Get inode number for anonymous file + struct stat stat_buf; + if (fstat(fd_anon, &stat_buf) == -1) { + ZErrno err; + log_error(gc, init)("Failed to determine inode number for anonymous file (%s)", err.to_string()); + return -1; + } + + log_debug(gc, init)("Heap backed by file %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino); + + return fd_anon; + } + + log_debug(gc, init)("Falling back to open/unlink"); + + // Create file name + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s/%s.%d", path.get(), name, os::current_process_id()); + + // Create file + const int fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); + if (fd == -1) { + ZErrno err; + log_error(gc, init)("Failed to create file %s (%s)", filename, err.to_string()); + return -1; + } + + // Unlink file + if (unlink(filename) == -1) { + ZErrno err; + log_error(gc, init)("Failed to unlink file %s (%s)", filename, err.to_string()); + return -1; + } + + log_debug(gc, init)("Heap backed by file %s", filename); + + return fd; +} + +int ZBackingFile::create_fd(const char* name) const { + if (ZPath == NULL) { + // If the path is not explicitly specified, then we first try to create a memfd file + // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might + // not be supported at all (requires kernel >= 3.17), or it might not support large + // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a + // file on an accessible tmpfs or hugetlbfs mount point. + const int fd = create_mem_fd(name); + if (fd != -1) { + return fd; + } + + log_debug(gc, init)("Falling back to searching for an accessible moint point"); + } + + return create_file_fd(name); +} + +bool ZBackingFile::is_initialized() const { + return _initialized; +} + +int ZBackingFile::fd() const { + return _fd; +} + +bool ZBackingFile::is_tmpfs() const { + return _filesystem == TMPFS_MAGIC; +} + +bool ZBackingFile::is_hugetlbfs() const { + return _filesystem == HUGETLBFS_MAGIC; +} + +bool ZBackingFile::tmpfs_supports_transparent_huge_pages() const { + // If the shmem_enabled file exists and is readable then we + // know the kernel supports transparent huge pages for tmpfs. + return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0; +} + +bool ZBackingFile::try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const { + // Try first smaller part. + const size_t offset0 = offset; + const size_t length0 = align_up(length / 2, alignment); + if (!try_expand_tmpfs(offset0, length0, alignment)) { + return false; + } + + // Try second smaller part. + const size_t offset1 = offset0 + length0; + const size_t length1 = length - length0; + if (!try_expand_tmpfs(offset1, length1, alignment)) { + return false; + } + + return true; +} + +bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const { + assert(length > 0, "Invalid length"); + assert(is_aligned(length, alignment), "Invalid length"); + + ZErrno err = posix_fallocate(_fd, offset, length); + + if (err == EINTR && length > alignment) { + // Calling posix_fallocate() with a large length can take a long + // time to complete. When running profilers, such as VTune, this + // syscall will be constantly interrupted by signals. Expanding + // the file in smaller steps avoids this problem. + return try_split_and_expand_tmpfs(offset, length, alignment); + } + + if (err) { + log_error(gc)("Failed to allocate backing file (%s)", err.to_string()); + return false; + } + + return true; +} + +bool ZBackingFile::expand_tmpfs(size_t offset, size_t length) const { + assert(is_tmpfs(), "Wrong filesystem"); + return try_expand_tmpfs(offset, length, os::vm_page_size()); +} + +bool ZBackingFile::expand_hugetlbfs(size_t offset, size_t length) const { + assert(is_hugetlbfs(), "Wrong filesystem"); + + // Prior to kernel 4.3, hugetlbfs did not support posix_fallocate(). + // Instead of posix_fallocate() we can use a well-known workaround, + // which involves truncating the file to requested size and then try + // to map it to verify that there are enough huge pages available to + // back it. + while (ftruncate(_fd, offset + length) == -1) { + ZErrno err; + if (err != EINTR) { + log_error(gc)("Failed to truncate backing file (%s)", err.to_string()); + return false; + } + } + + // If we fail mapping during initialization, i.e. when we are pre-mapping + // the heap, then we wait and retry a few times before giving up. Otherwise + // there is a risk that running JVMs back-to-back will fail, since there + // is a delay between process termination and the huge pages owned by that + // process being returned to the huge page pool and made available for new + // allocations. + void* addr = MAP_FAILED; + const int max_attempts = 3; + for (int attempt = 1; attempt <= max_attempts; attempt++) { + addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset); + if (addr != MAP_FAILED || is_init_completed()) { + // Mapping was successful or initialization phase has completed + break; + } + + ZErrno err; + log_debug(gc)("Failed to map backing file (%s), attempt %d of %d", + err.to_string(), attempt, max_attempts); + + // Wait and retry in one second, in the hope that + // huge pages will be available by then. + sleep(1); + } + + if (addr == MAP_FAILED) { + // Not enough huge pages left + ZErrno err; + log_error(gc)("Failed to map backing file (%s)", err.to_string()); + return false; + } + + // Successful mapping, unmap again. From now on the pages we mapped + // will be reserved for this file. + if (munmap(addr, length) == -1) { + ZErrno err; + log_error(gc)("Failed to unmap backing file (%s)", err.to_string()); + return false; + } + + return true; +} + +bool ZBackingFile::expand(size_t offset, size_t length) const { + return is_hugetlbfs() ? expand_hugetlbfs(offset, length) : expand_tmpfs(offset, length); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zBackingFile_linux_x86.hpp 2018-06-01 22:30:11.022603780 +0200 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_X86_ZBACKINGFILE_LINUX_X86_HPP +#define OS_CPU_LINUX_X86_ZBACKINGFILE_LINUX_X86_HPP + +#include "memory/allocation.hpp" + +class ZBackingFile { +private: + int _fd; + uint64_t _filesystem; + bool _initialized; + + int create_mem_fd(const char* name) const; + int create_file_fd(const char* name) const; + int create_fd(const char* name) const; + + bool is_tmpfs() const; + bool is_hugetlbfs() const; + bool tmpfs_supports_transparent_huge_pages() const; + + bool try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const; + bool try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const; + bool expand_tmpfs(size_t offset, size_t length) const; + + bool expand_hugetlbfs(size_t offset, size_t length) const; + +public: + ZBackingFile(); + + bool is_initialized() const; + + int fd() const; + bool expand(size_t offset, size_t length) const; +}; + +#endif // OS_CPU_LINUX_X86_ZBACKINGFILE_LINUX_X86_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zBackingPath_linux_x86.cpp 2018-06-01 22:30:11.406620356 +0200 @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zArray.inline.hpp" +#include "gc/z/zBackingPath_linux_x86.hpp" +#include "gc/z/zErrno.hpp" +#include "gc/z/zFlags.hpp" +#include "logging/log.hpp" + +#include +#include + +// Mount information, see proc(5) for more details. +#define PROC_SELF_MOUNTINFO "/proc/self/mountinfo" + +ZBackingPath::ZBackingPath(const char* filesystem, const char* preferred_path) { + if (ZPath != NULL) { + // Use specified path + _path = strdup(ZPath); + } else { + // Find suitable path + _path = find_mountpoint(filesystem, preferred_path); + } +} + +ZBackingPath::~ZBackingPath() { + free(_path); + _path = NULL; +} + +char* ZBackingPath::get_mountpoint(const char* line, const char* filesystem) const { + char* line_mountpoint = NULL; + char* line_filesystem = NULL; + + // Parse line and return a newly allocated string containing the mountpoint if + // the line contains a matching filesystem and the mountpoint is accessible by + // the current user. + if (sscanf(line, "%*u %*u %*u:%*u %*s %ms %*[^-]- %ms", &line_mountpoint, &line_filesystem) != 2 || + strcmp(line_filesystem, filesystem) != 0 || + access(line_mountpoint, R_OK|W_OK|X_OK) != 0) { + // Not a matching or accessible filesystem + free(line_mountpoint); + line_mountpoint = NULL; + } + + free(line_filesystem); + + return line_mountpoint; +} + +void ZBackingPath::get_mountpoints(ZArray* mountpoints, const char* filesystem) const { + FILE* fd = fopen(PROC_SELF_MOUNTINFO, "r"); + if (fd == NULL) { + ZErrno err; + log_error(gc, init)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string()); + return; + } + + char* line = NULL; + size_t length = 0; + + while (getline(&line, &length, fd) != -1) { + char* const mountpoint = get_mountpoint(line, filesystem); + if (mountpoint != NULL) { + mountpoints->add(mountpoint); + } + } + + free(line); + fclose(fd); +} + +void ZBackingPath::free_mountpoints(ZArray* mountpoints) const { + ZArrayIterator iter(mountpoints); + for (char* mountpoint; iter.next(&mountpoint);) { + free(mountpoint); + } + mountpoints->clear(); +} + +char* ZBackingPath::find_mountpoint(const char* filesystem, const char* preferred_mountpoint) const { + char* path = NULL; + ZArray mountpoints; + + get_mountpoints(&mountpoints, filesystem); + + if (mountpoints.size() == 0) { + // No filesystem found + log_error(gc, init)("Failed to find an accessible %s filesystem", filesystem); + } else if (mountpoints.size() == 1) { + // One filesystem found + path = strdup(mountpoints.at(0)); + } else if (mountpoints.size() > 1) { + // More than one filesystem found + ZArrayIterator iter(&mountpoints); + for (char* mountpoint; iter.next(&mountpoint);) { + if (!strcmp(mountpoint, preferred_mountpoint)) { + // Preferred mount point found + path = strdup(mountpoint); + break; + } + } + + if (path == NULL) { + // Preferred mount point not found + log_error(gc, init)("More than one %s filesystem found:", filesystem); + ZArrayIterator iter2(&mountpoints); + for (char* mountpoint; iter2.next(&mountpoint);) { + log_error(gc, init)(" %s", mountpoint); + } + } + } + + free_mountpoints(&mountpoints); + + return path; +} + +const char* ZBackingPath::get() const { + return _path; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zBackingPath_linux_x86.hpp 2018-06-01 22:30:11.804637536 +0200 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_X86_ZBACKINGPATH_LINUX_X86_HPP +#define OS_CPU_LINUX_X86_ZBACKINGPATH_LINUX_X86_HPP + +#include "gc/z/zArray.hpp" +#include "memory/allocation.hpp" + +class ZBackingPath : public StackObj { +private: + char* _path; + + char* get_mountpoint(const char* line, const char* filesystem) const; + void get_mountpoints(ZArray* mountpoints, const char* filesystem) const; + void free_mountpoints(ZArray* mountpoints) const; + char* find_mountpoint(const char* filesystem, const char* preferred_mountpoint) const; + +public: + ZBackingPath(const char* filesystem, const char* preferred_path); + ~ZBackingPath(); + + const char* get() const; +}; + +#endif // OS_CPU_LINUX_X86_ZBACKINGPATH_LINUX_X86_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zGlobals_linux_x86.cpp 2018-06-01 22:30:12.198654543 +0200 @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zGlobals.hpp" + +uintptr_t ZAddressReservedStart() { + return ZAddressMetadataMarked0; +} + +uintptr_t ZAddressReservedEnd() { + return ZAddressMetadataRemapped + ZAddressOffsetMax; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zGlobals_linux_x86.hpp 2018-06-01 22:30:12.590671464 +0200 @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_X86_ZGLOBALS_LINUX_X86_HPP +#define OS_CPU_LINUX_X86_ZGLOBALS_LINUX_X86_HPP + +// +// Page Allocation Tiers +// --------------------- +// +// Page Type Page Size Object Size Limit Object Alignment +// ------------------------------------------------------------------ +// Small 2M <= 265K +// Medium 32M <= 4M 4K +// Large X*M > 4M 2M +// ------------------------------------------------------------------ +// +// +// Address Space & Pointer Layout +// ------------------------------ +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000140000000000 (20TB) +// | Remapped View | +// +--------------------------------+ 0x0000100000000000 (16TB) +// | (Reserved, but unused) | +// +--------------------------------+ 0x00000c0000000000 (12TB) +// | Marked1 View | +// +--------------------------------+ 0x0000080000000000 (8TB) +// | Marked0 View | +// +--------------------------------+ 0x0000040000000000 (4TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// +// 6 4 4 4 4 4 0 +// 3 7 6 5 2 1 0 +// +-------------------+-+----+-----------------------------------------------+ +// |00000000 00000000 0|0|1111|11 11111111 11111111 11111111 11111111 11111111| +// +-------------------+-+----+-----------------------------------------------+ +// | | | | +// | | | * 41-0 Object Offset (42-bits, 4TB address space) +// | | | +// | | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB) +// | | 0010 = Marked1 (Address view 8-12TB) +// | | 0100 = Remapped (Address view 16-20TB) +// | | 1000 = Finalizable (Address view N/A) +// | | +// | * 46-46 Unused (1-bit, always zero) +// | +// * 63-47 Fixed (17-bits, always zero) +// + +const size_t ZPlatformPageSizeSmallShift = 21; // 2M + +const size_t ZPlatformAddressOffsetBits = 42; // 4TB + +const uintptr_t ZPlatformAddressMetadataShift = ZPlatformAddressOffsetBits; + +const uintptr_t ZPlatformAddressSpaceStart = (uintptr_t)1 << ZPlatformAddressOffsetBits; +const uintptr_t ZPlatformAddressSpaceSize = ((uintptr_t)1 << ZPlatformAddressOffsetBits) * 4; + +const size_t ZPlatformCacheLineSize = 64; + +#endif // OS_CPU_LINUX_X86_ZGLOBALS_LINUX_X86_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zLargePages_linux_x86.cpp 2018-06-01 22:30:12.962687522 +0200 @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zLargePages.hpp" +#include "runtime/globals.hpp" + +void ZLargePages::initialize_platform() { + if (UseLargePages) { + if (UseTransparentHugePages) { + _state = Transparent; + } else { + _state = Explicit; + } + } else { + _state = Disabled; + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zNUMA_linux_x86.cpp 2018-06-01 22:30:13.347704141 +0200 @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "gc/z/zErrno.hpp" +#include "gc/z/zCPU.hpp" +#include "gc/z/zNUMA.hpp" +#include "runtime/os.hpp" +#include "utilities/debug.hpp" + +#include +#include + +#ifndef MPOL_F_NODE +#define MPOL_F_NODE (1<<0) /* return next IL mode instead of node mask */ +#endif + +#ifndef MPOL_F_ADDR +#define MPOL_F_ADDR (1<<1) /* look up vma using address */ +#endif + +static int z_get_mempolicy(uint32_t* mode, const unsigned long *nmask, unsigned long maxnode, uintptr_t addr, int flags) { + return syscall(__NR_get_mempolicy, mode, nmask, maxnode, addr, flags); +} + +void ZNUMA::initialize_platform() { + _enabled = UseNUMA; +} + +uint32_t ZNUMA::count() { + if (!_enabled) { + // NUMA support not enabled + return 1; + } + + return os::Linux::numa_max_node() + 1; +} + +uint32_t ZNUMA::id() { + if (!_enabled) { + // NUMA support not enabled + return 0; + } + + return os::Linux::get_node_by_cpu(ZCPU::id()); +} + +uint32_t ZNUMA::memory_id(uintptr_t addr) { + if (!_enabled) { + // NUMA support not enabled, assume everything belongs to node zero + return 0; + } + + uint32_t id = (uint32_t)-1; + + if (z_get_mempolicy(&id, NULL, 0, addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) { + ZErrno err; + fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string()); + } + + assert(id < count(), "Invalid NUMA id"); + + return id; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.cpp 2018-06-01 22:30:13.736720932 +0200 @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zBackingFile_linux_x86.hpp" +#include "gc/z/zErrno.hpp" +#include "gc/z/zFlags.hpp" +#include "gc/z/zLargePages.inline.hpp" +#include "gc/z/zMemory.hpp" +#include "gc/z/zNUMA.hpp" +#include "gc/z/zPhysicalMemory.inline.hpp" +#include "gc/z/zPhysicalMemoryBacking_linux_x86.hpp" +#include "logging/log.hpp" +#include "runtime/os.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +#include +#include +#include + +// Support for building on older Linux systems +#ifndef MADV_HUGEPAGE +#define MADV_HUGEPAGE 14 +#endif + +// Proc file entry for max map mount +#define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count" + +ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity, size_t granule_size) : + _manager(), + _file(), + _granule_size(granule_size) { + + // Check and warn if max map count seems too low + check_max_map_count(max_capacity, granule_size); +} + +void ZPhysicalMemoryBacking::check_max_map_count(size_t max_capacity, size_t granule_size) const { + const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT; + FILE* const file = fopen(filename, "r"); + if (file == NULL) { + // Failed to open file, skip check + log_debug(gc)("Failed to open %s", filename); + return; + } + + size_t actual_max_map_count = 0; + const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count); + fclose(file); + if (result != 1) { + // Failed to read file, skip check + log_debug(gc)("Failed to read %s", filename); + return; + } + + // The required max map count is impossible to calculate exactly since subsystems + // other than ZGC are also creating memory mappings, and we have no control over that. + // However, ZGC tends to create the most mappings and dominate the total count. + // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. + // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. + const size_t required_max_map_count = (max_capacity / granule_size) * 3 * 1.2; + if (actual_max_map_count < required_max_map_count) { + log_warning(gc)("The system limit on number of memory mappings " + "per process might be too low for the given"); + log_warning(gc)("Java heap size (" SIZE_FORMAT "M). Please " + "adjust %s to allow for at least", max_capacity / M, filename); + log_warning(gc)(SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). " + "Continuing execution with the current limit could", + required_max_map_count, actual_max_map_count); + log_warning(gc)("lead to a fatal error down the line, due to failed " + "attempts to map memory."); + } +} + +bool ZPhysicalMemoryBacking::is_initialized() const { + return _file.is_initialized(); +} + +bool ZPhysicalMemoryBacking::expand(size_t from, size_t to) { + const size_t size = to - from; + + // Expand + if (!_file.expand(from, size)) { + return false; + } + + // Add expanded space to free list + _manager.free(from, size); + + return true; +} + +ZPhysicalMemory ZPhysicalMemoryBacking::alloc(size_t size) { + assert(is_aligned(size, _granule_size), "Invalid size"); + + ZPhysicalMemory pmem; + + // Allocate segments + for (size_t allocated = 0; allocated < size; allocated += _granule_size) { + const uintptr_t start = _manager.alloc_from_front(_granule_size); + assert(start != UINTPTR_MAX, "Allocation should never fail"); + pmem.add_segment(ZPhysicalMemorySegment(start, _granule_size)); + } + + return pmem; +} + +void ZPhysicalMemoryBacking::free(ZPhysicalMemory pmem) { + const size_t nsegments = pmem.nsegments(); + + // Free segments + for (size_t i = 0; i < nsegments; i++) { + const ZPhysicalMemorySegment segment = pmem.segment(i); + _manager.free(segment.start(), segment.size()); + } +} + +void ZPhysicalMemoryBacking::map_failed(ZErrno err) const { + if (err == ENOMEM) { + fatal("Failed to map memory. Please check the system limit on number of " + "memory mappings allowed per process (see %s)", ZFILENAME_PROC_MAX_MAP_COUNT); + } else { + fatal("Failed to map memory (%s)", err.to_string()); + } +} + +void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size) const { + if (madvise((void*)addr, size, MADV_HUGEPAGE) == -1) { + ZErrno err; + log_error(gc)("Failed to advise use of transparent huge pages (%s)", err.to_string()); + } +} + +void ZPhysicalMemoryBacking::pretouch_view(uintptr_t addr, size_t size) const { + const size_t page_size = ZLargePages::is_explicit() ? os::large_page_size() : os::vm_page_size(); + os::pretouch_memory((void*)addr, (void*)(addr + size), page_size); +} + +void ZPhysicalMemoryBacking::map_view(ZPhysicalMemory pmem, uintptr_t addr, bool pretouch) const { + const size_t nsegments = pmem.nsegments(); + + // Map segments + for (size_t i = 0; i < nsegments; i++) { + const ZPhysicalMemorySegment segment = pmem.segment(i); + const size_t size = segment.size(); + const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start()); + if (res == MAP_FAILED) { + ZErrno err; + map_failed(err); + } + + // Advise on use of transparent huge pages before touching it + if (ZLargePages::is_transparent()) { + advise_view(addr, size); + } + + // NUMA interleave memory before touching it + ZNUMA::memory_interleave(addr, size); + + if (pretouch) { + pretouch_view(addr, size); + } + + addr += size; + } +} + +void ZPhysicalMemoryBacking::unmap_view(ZPhysicalMemory pmem, uintptr_t addr) const { + // Note that we must keep the address space reservation intact and just detach + // the backing memory. For this reason we map a new anonymous, non-accessible + // and non-reserved page over the mapping instead of actually unmapping. + const size_t size = pmem.size(); + const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); + if (res == MAP_FAILED) { + ZErrno err; + map_failed(err); + } +} + +uintptr_t ZPhysicalMemoryBacking::nmt_address(uintptr_t offset) const { + // From an NMT point of view we treat the first heap mapping (marked0) as committed + return ZAddress::marked0(offset); +} + +void ZPhysicalMemoryBacking::map(ZPhysicalMemory pmem, uintptr_t offset) const { + if (ZUnmapBadViews) { + // Only map the good view, for debugging only + map_view(pmem, ZAddress::good(offset), AlwaysPreTouch); + } else { + // Map all views + map_view(pmem, ZAddress::marked0(offset), AlwaysPreTouch); + map_view(pmem, ZAddress::marked1(offset), AlwaysPreTouch); + map_view(pmem, ZAddress::remapped(offset), AlwaysPreTouch); + } +} + +void ZPhysicalMemoryBacking::unmap(ZPhysicalMemory pmem, uintptr_t offset) const { + if (ZUnmapBadViews) { + // Only map the good view, for debugging only + unmap_view(pmem, ZAddress::good(offset)); + } else { + // Unmap all views + unmap_view(pmem, ZAddress::marked0(offset)); + unmap_view(pmem, ZAddress::marked1(offset)); + unmap_view(pmem, ZAddress::remapped(offset)); + } +} + +void ZPhysicalMemoryBacking::flip(ZPhysicalMemory pmem, uintptr_t offset) const { + assert(ZUnmapBadViews, "Should be enabled"); + const uintptr_t addr_good = ZAddress::good(offset); + const uintptr_t addr_bad = ZAddress::is_marked(ZAddressGoodMask) ? ZAddress::remapped(offset) : ZAddress::marked(offset); + // Map/Unmap views + map_view(pmem, addr_good, false /* pretouch */); + unmap_view(pmem, addr_bad); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zPhysicalMemoryBacking_linux_x86.hpp 2018-06-01 22:30:14.128737853 +0200 @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_X86_ZPHYSICALMEMORYBACKING_LINUX_X86_HPP +#define OS_CPU_LINUX_X86_ZPHYSICALMEMORYBACKING_LINUX_X86_HPP + +#include "gc/z/zBackingFile_linux_x86.hpp" +#include "gc/z/zMemory.hpp" + +class ZErrno; +class ZPhysicalMemory; + +class ZPhysicalMemoryBacking { +private: + ZMemoryManager _manager; + ZBackingFile _file; + const size_t _granule_size; + + void check_max_map_count(size_t max_capacity, size_t granule_size) const; + void map_failed(ZErrno err) const; + + void advise_view(uintptr_t addr, size_t size) const; + void pretouch_view(uintptr_t addr, size_t size) const; + void map_view(ZPhysicalMemory pmem, uintptr_t addr, bool pretouch) const; + void unmap_view(ZPhysicalMemory pmem, uintptr_t addr) const; + +public: + ZPhysicalMemoryBacking(size_t max_capacity, size_t granule_size); + + bool is_initialized() const; + + bool expand(size_t from, size_t to); + ZPhysicalMemory alloc(size_t size); + void free(ZPhysicalMemory pmem); + + uintptr_t nmt_address(uintptr_t offset) const; + + void map(ZPhysicalMemory pmem, uintptr_t offset) const; + void unmap(ZPhysicalMemory pmem, uintptr_t offset) const; + void flip(ZPhysicalMemory pmem, uintptr_t offset) const; +}; + +#endif // OS_CPU_LINUX_X86_ZPHYSICALMEMORYBACKING_LINUX_X86_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/os_cpu/linux_x86/gc/z/zVirtualMemory_linux_x86.cpp 2018-06-01 22:30:14.516754602 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zVirtualMemory.hpp" +#include "logging/log.hpp" + +#include +#include + +bool ZVirtualMemoryManager::reserve(uintptr_t start, size_t size) { + // Reserve address space + const uintptr_t actual_start = (uintptr_t)mmap((void*)start, size, PROT_NONE, + MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); + if (actual_start != start) { + log_error(gc)("Failed to reserve address space for Java heap"); + return false; + } + + return true; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp 2018-06-01 22:30:14.899771134 +0200 @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "c1/c1_LIR.hpp" +#include "c1/c1_LIRGenerator.hpp" +#include "c1/c1_CodeStubs.hpp" +#include "gc/z/c1/zBarrierSetC1.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" +#include "gc/z/zThreadLocalData.hpp" +#include "utilities/macros.hpp" + +ZLoadBarrierStubC1::ZLoadBarrierStubC1(LIRAccess& access, LIR_Opr ref, address runtime_stub) : + _decorators(access.decorators()), + _ref_addr(access.resolved_addr()), + _ref(ref), + _tmp(LIR_OprFact::illegalOpr), + _patch_info(access.patch_emit_info()), + _runtime_stub(runtime_stub) { + + // Allocate tmp register if needed + if (!_ref_addr->is_register()) { + assert(_ref_addr->is_address(), "Must be an address"); + if (_ref_addr->as_address_ptr()->index()->is_valid() || + _ref_addr->as_address_ptr()->disp() != 0) { + // Has index or displacement, need tmp register to load address into + _tmp = access.gen()->new_pointer_register(); + } else { + // No index or displacement, address available in base register + _ref_addr = _ref_addr->as_address_ptr()->base(); + } + } + + assert(_ref->is_register(), "Must be a register"); + assert(_ref_addr->is_register() != _tmp->is_register(), "Only one should be a register"); +} + +DecoratorSet ZLoadBarrierStubC1::decorators() const { + return _decorators; +} + +LIR_Opr ZLoadBarrierStubC1::ref() const { + return _ref; +} + +LIR_Opr ZLoadBarrierStubC1::ref_addr() const { + return _ref_addr; +} + +LIR_Opr ZLoadBarrierStubC1::tmp() const { + return _tmp; +} + +LIR_PatchCode ZLoadBarrierStubC1::patch_code() const { + return (_decorators & C1_NEEDS_PATCHING) != 0 ? lir_patch_normal : lir_patch_none; +} + +CodeEmitInfo*& ZLoadBarrierStubC1::patch_info() { + return _patch_info; +} + +address ZLoadBarrierStubC1::runtime_stub() const { + return _runtime_stub; +} + +void ZLoadBarrierStubC1::visit(LIR_OpVisitState* visitor) { + if (_patch_info != NULL) { + visitor->do_slow_case(_patch_info); + } else { + visitor->do_slow_case(); + } + + visitor->do_input(_ref_addr); + visitor->do_output(_ref); + + if (_tmp->is_valid()) { + visitor->do_temp(_tmp); + } +} + +void ZLoadBarrierStubC1::emit_code(LIR_Assembler* ce) { + ZBarrierSet::assembler()->generate_c1_load_barrier_stub(ce, this); +} + +#ifndef PRODUCT +void ZLoadBarrierStubC1::print_name(outputStream* out) const { + out->print("ZLoadBarrierStubC1"); +} +#endif // PRODUCT + +class LIR_OpZLoadBarrierTest : public LIR_Op { +private: + LIR_Opr _opr; + +public: + LIR_OpZLoadBarrierTest(LIR_Opr opr) : + LIR_Op(), + _opr(opr) {} + + virtual void visit(LIR_OpVisitState* state) { + state->do_input(_opr); + } + + virtual void emit_code(LIR_Assembler* ce) { + ZBarrierSet::assembler()->generate_c1_load_barrier_test(ce, _opr); + } + + virtual void print_instr(outputStream* out) const { + _opr->print(out); + out->print(" "); + } + +#ifndef PRODUCT + virtual const char* name() const { + return "lir_z_load_barrier_test"; + } +#endif // PRODUCT +}; + +static bool barrier_needed(LIRAccess& access) { + return ZBarrierSet::barrier_needed(access.decorators(), access.type()); +} + +ZBarrierSetC1::ZBarrierSetC1() : + _load_barrier_on_oop_field_preloaded_runtime_stub(NULL), + _load_barrier_on_weak_oop_field_preloaded_runtime_stub(NULL) {} + +address ZBarrierSetC1::load_barrier_on_oop_field_preloaded_runtime_stub(DecoratorSet decorators) const { + assert((decorators & ON_PHANTOM_OOP_REF) == 0, "Unsupported decorator"); + //assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Unsupported decorator"); + + if ((decorators & ON_WEAK_OOP_REF) != 0) { + return _load_barrier_on_weak_oop_field_preloaded_runtime_stub; + } else { + return _load_barrier_on_oop_field_preloaded_runtime_stub; + } +} + +#ifdef ASSERT +#define __ access.gen()->lir(__FILE__, __LINE__)-> +#else +#define __ access.gen()->lir()-> +#endif + +void ZBarrierSetC1::load_barrier(LIRAccess& access, LIR_Opr result) const { + // Fast path + __ append(new LIR_OpZLoadBarrierTest(result)); + + // Slow path + const address runtime_stub = load_barrier_on_oop_field_preloaded_runtime_stub(access.decorators()); + CodeStub* const stub = new ZLoadBarrierStubC1(access, result, runtime_stub); + __ branch(lir_cond_notEqual, T_ADDRESS, stub); + __ branch_destination(stub->continuation()); +} + +#undef __ + +void ZBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { + BarrierSetC1::load_at_resolved(access, result); + + if (barrier_needed(access)) { + load_barrier(access, result); + } +} + +static void pre_load_barrier(LIRAccess& access) { + DecoratorSet decorators = access.decorators(); + + // Downgrade access to MO_UNORDERED + decorators = (decorators & ~MO_DECORATOR_MASK) | MO_UNORDERED; + + // Remove C1_WRITE_ACCESS + decorators = (decorators & ~C1_WRITE_ACCESS); + + // Generate synthetic load at + access.gen()->access_load_at(decorators, + access.type(), + access.base().item(), + access.offset().opr(), + access.gen()->new_register(access.type()), + NULL /* patch_emit_info */, + NULL /* load_emit_info */); +} + +LIR_Opr ZBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) { + if (barrier_needed(access)) { + pre_load_barrier(access); + } + + return BarrierSetC1::atomic_xchg_at_resolved(access, value); +} + +LIR_Opr ZBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) { + if (barrier_needed(access)) { + pre_load_barrier(access); + } + + return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); +} + +class ZLoadBarrierRuntimeStubCodeGenClosure : public StubAssemblerCodeGenClosure { +private: + const DecoratorSet _decorators; + +public: + ZLoadBarrierRuntimeStubCodeGenClosure(DecoratorSet decorators) : + _decorators(decorators) {} + + virtual OopMapSet* generate_code(StubAssembler* sasm) { + ZBarrierSet::assembler()->generate_c1_load_barrier_runtime_stub(sasm, _decorators); + return NULL; + } +}; + +static address generate_c1_runtime_stub(BufferBlob* blob, DecoratorSet decorators, const char* name) { + ZLoadBarrierRuntimeStubCodeGenClosure cl(decorators); + CodeBlob* const code_blob = Runtime1::generate_blob(blob, -1 /* stub_id */, name, false /* expect_oop_map*/, &cl); + return code_blob->code_begin(); +} + +void ZBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* blob) { + _load_barrier_on_oop_field_preloaded_runtime_stub = + generate_c1_runtime_stub(blob, ON_STRONG_OOP_REF, "load_barrier_on_oop_field_preloaded_runtime_stub"); + _load_barrier_on_weak_oop_field_preloaded_runtime_stub = + generate_c1_runtime_stub(blob, ON_WEAK_OOP_REF, "load_barrier_on_weak_oop_field_preloaded_runtime_stub"); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/c1/zBarrierSetC1.hpp 2018-06-01 22:30:15.282787667 +0200 @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_C1_ZBARRIERSETC1_HPP +#define SHARE_GC_Z_C1_ZBARRIERSETC1_HPP + +#include "c1/c1_CodeStubs.hpp" +#include "c1/c1_IR.hpp" +#include "c1/c1_LIR.hpp" +#include "gc/shared/c1/barrierSetC1.hpp" +#include "oops/accessDecorators.hpp" + +class ZLoadBarrierStubC1 : public CodeStub { +private: + DecoratorSet _decorators; + LIR_Opr _ref_addr; + LIR_Opr _ref; + LIR_Opr _tmp; + CodeEmitInfo* _patch_info; + address _runtime_stub; + +public: + ZLoadBarrierStubC1(LIRAccess& access, LIR_Opr ref, address runtime_stub); + + DecoratorSet decorators() const; + LIR_Opr ref() const; + LIR_Opr ref_addr() const; + LIR_Opr tmp() const; + LIR_PatchCode patch_code() const; + CodeEmitInfo*& patch_info(); + address runtime_stub() const; + + virtual void emit_code(LIR_Assembler* ce); + virtual void visit(LIR_OpVisitState* visitor); + +#ifndef PRODUCT + virtual void print_name(outputStream* out) const; +#endif // PRODUCT +}; + +class ZBarrierSetC1 : public BarrierSetC1 { +private: + address _load_barrier_on_oop_field_preloaded_runtime_stub; + address _load_barrier_on_weak_oop_field_preloaded_runtime_stub; + + address load_barrier_on_oop_field_preloaded_runtime_stub(DecoratorSet decorators) const; + void load_barrier(LIRAccess& access, LIR_Opr result) const; + +protected: + virtual void load_at_resolved(LIRAccess& access, LIR_Opr result); + virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value); + virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value); + +public: + ZBarrierSetC1(); + + virtual void generate_c1_runtime_stubs(BufferBlob* blob); +}; + +#endif // SHARE_GC_Z_C1_ZBARRIERSETC1_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp 2018-06-01 22:30:15.665804200 +0200 @@ -0,0 +1,1491 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "opto/compile.hpp" +#include "opto/castnode.hpp" +#include "opto/graphKit.hpp" +#include "opto/idealKit.hpp" +#include "opto/loopnode.hpp" +#include "opto/macro.hpp" +#include "opto/node.hpp" +#include "opto/type.hpp" +#include "utilities/macros.hpp" +#include "gc/z/c2/zBarrierSetC2.hpp" +#include "gc/z/zThreadLocalData.hpp" +#include "gc/z/zBarrierSetRuntime.hpp" + +ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) + : _load_barrier_nodes(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) {} + +int ZBarrierSetC2State::load_barrier_count() const { + return _load_barrier_nodes->length(); +} + +void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) { + assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list"); + _load_barrier_nodes->append(n); +} + +void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) { + // this function may be called twice for a node so check + // that the node is in the array before attempting to remove it + if (_load_barrier_nodes->contains(n)) { + _load_barrier_nodes->remove(n); + } +} + +LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const { + return _load_barrier_nodes->at(idx); +} + +void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { + return new(comp_arena) ZBarrierSetC2State(comp_arena); +} + +ZBarrierSetC2State* ZBarrierSetC2::state() const { + return reinterpret_cast(Compile::current()->barrier_set_state()); +} + +bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { + return node->is_LoadBarrier(); +} + +void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { + if (node->is_LoadBarrier()) { + state()->add_load_barrier_node(node->as_LoadBarrier()); + } +} + +void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { + if (node->is_LoadBarrier()) { + state()->remove_load_barrier_node(node->as_LoadBarrier()); + } +} + +void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { + // Remove useless LoadBarrier nodes + ZBarrierSetC2State* s = state(); + for (int i = s->load_barrier_count()-1; i >= 0; i--) { + LoadBarrierNode* n = s->load_barrier_node(i); + if (!useful.member(n)) { + unregister_potential_barrier_node(n); + } + } +} + +void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { + if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { + worklist.push(node); + } +} + +void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) { + // Look for dominating barriers on the same address only once all + // other loop opts are over: loop opts may cause a safepoint to be + // inserted between a barrier and its dominating barrier. + Compile* C = Compile::current(); + ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2(); + ZBarrierSetC2State* s = bs->state(); + if (s->load_barrier_count() >= 2) { + Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]); + PhaseIdealLoop ideal_loop(igvn, true, false, true); + if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2); + } +} + +void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const { + // Permanent temporary workaround + // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is + // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to + // the worklist. Unless we add them explicitly they are not guaranteed to end up there. + ZBarrierSetC2State* s = state(); + + for (int i = 0; i < s->load_barrier_count(); i++) { + LoadBarrierNode* n = s->load_barrier_node(i); + worklist->push(n); + } +} + +const TypeFunc* ZBarrierSetC2::load_barrier_Type() const { + const Type** fields; + + // Create input types (domain) + fields = TypeTuple::fields(2); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; + fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM; + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); + + // Create result type (range) + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM; + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); + + return TypeFunc::make(domain, range); +} + +// == LoadBarrierNode == + +LoadBarrierNode::LoadBarrierNode(Compile* C, + Node* c, + Node* mem, + Node* val, + Node* adr, + bool weak, + bool writeback, + bool oop_reload_allowed) : + MultiNode(Number_of_Inputs), + _weak(weak), + _writeback(writeback), + _oop_reload_allowed(oop_reload_allowed) { + init_req(Control, c); + init_req(Memory, mem); + init_req(Oop, val); + init_req(Address, adr); + init_req(Similar, C->top()); + + init_class_id(Class_LoadBarrier); + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + bs->register_potential_barrier_node(this); +} + +const Type *LoadBarrierNode::bottom_type() const { + const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); + Node* in_oop = in(Oop); + floadbarrier[Control] = Type::CONTROL; + floadbarrier[Memory] = Type::MEMORY; + floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type(); + return TypeTuple::make(Number_of_Outputs, floadbarrier); +} + +const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { + const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); + const Type* val_t = phase->type(in(Oop)); + floadbarrier[Control] = Type::CONTROL; + floadbarrier[Memory] = Type::MEMORY; + floadbarrier[Oop] = val_t; + return TypeTuple::make(Number_of_Outputs, floadbarrier); +} + +bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { + if (phase != NULL) { + return phase->is_dominator(d, n); + } + + for (int i = 0; i < 10 && n != NULL; i++) { + n = IfNode::up_one_dom(n, linear_only); + if (n == d) { + return true; + } + } + + return false; +} + +LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { + Node* val = in(LoadBarrierNode::Oop); + if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { + LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); + assert(lb->in(Address) == in(Address), ""); + // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier. + if (lb->in(Oop) == in(Oop)) { + return lb; + } + // Follow chain of load barrier through Similar edges + while (!lb->in(Similar)->is_top()) { + lb = lb->in(Similar)->in(0)->as_LoadBarrier(); + assert(lb->in(Address) == in(Address), ""); + } + if (lb != in(Similar)->in(0)) { + return lb; + } + } + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { + Node* u = val->fast_out(i); + if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) { + Node* this_ctrl = in(LoadBarrierNode::Control); + Node* other_ctrl = u->in(LoadBarrierNode::Control); + if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { + return u->as_LoadBarrier(); + } + } + } + + if (ZVerifyLoadBarriers || can_be_eliminated()) { + return NULL; + } + + if (!look_for_similar) { + return NULL; + } + + Node* addr = in(LoadBarrierNode::Address); + for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { + Node* u = addr->fast_out(i); + if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { + Node* this_ctrl = in(LoadBarrierNode::Control); + Node* other_ctrl = u->in(LoadBarrierNode::Control); + if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { + ResourceMark rm; + Unique_Node_List wq; + wq.push(in(LoadBarrierNode::Control)); + bool ok = true; + bool dom_found = false; + for (uint next = 0; next < wq.size(); ++next) { + Node *n = wq.at(next); + if (n->is_top()) { + return NULL; + } + assert(n->is_CFG(), ""); + if (n->is_SafePoint()) { + ok = false; + break; + } + if (n == u) { + dom_found = true; + continue; + } + if (n->is_Region()) { + for (uint i = 1; i < n->req(); i++) { + Node* m = n->in(i); + if (m != NULL) { + wq.push(m); + } + } + } else { + Node* m = n->in(0); + if (m != NULL) { + wq.push(m); + } + } + } + if (ok) { + assert(dom_found, ""); + return u->as_LoadBarrier();; + } + break; + } + } + } + + return NULL; +} + +void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { + // change to that barrier may affect a dominated barrier so re-push those + Node* val = in(LoadBarrierNode::Oop); + + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { + Node* u = val->fast_out(i); + if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) { + Node* this_ctrl = in(Control); + Node* other_ctrl = u->in(Control); + if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { + igvn->_worklist.push(u); + } + } + + Node* addr = in(LoadBarrierNode::Address); + for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { + Node* u = addr->fast_out(i); + if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) { + Node* this_ctrl = in(Control); + Node* other_ctrl = u->in(Control); + if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { + igvn->_worklist.push(u); + } + } + } + } +} + +Node *LoadBarrierNode::Identity(PhaseGVN *phase) { + if (!phase->C->directive()->ZOptimizeLoadBarriersOption) { + return this; + } + + bool redundant_addr = false; + LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); + if (dominating_barrier != NULL) { + assert(dominating_barrier->in(Oop) == in(Oop), ""); + return dominating_barrier; + } + + return this; +} + +Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { + if (remove_dead_region(phase, can_reshape)) { + return this; + } + + Node* val = in(Oop); + Node* mem = in(Memory); + Node* ctrl = in(Control); + Node* adr = in(Address); + assert(val->Opcode() != Op_LoadN, ""); + + if (mem->is_MergeMem()) { + Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); + set_req(Memory, new_mem); + if (mem->outcnt() == 0 && can_reshape) { + phase->is_IterGVN()->_worklist.push(mem); + } + + return this; + } + + bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption; + LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL; + if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { + assert(in(Address) == dominating_barrier->in(Address), ""); + set_req(Similar, dominating_barrier->proj_out(Oop)); + return this; + } + + bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) || + (can_reshape && (dominating_barrier != NULL || !has_true_uses())); + + if (eliminate) { + if (can_reshape) { + PhaseIterGVN* igvn = phase->is_IterGVN(); + Node* out_ctrl = proj_out_or_null(Control); + Node* out_res = proj_out_or_null(Oop); + + if (out_ctrl != NULL) { + igvn->replace_node(out_ctrl, ctrl); + } + + // That transformation may cause the Similar edge on the load barrier to be invalid + fix_similar_in_uses(igvn); + if (out_res != NULL) { + if (dominating_barrier != NULL) { + igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); + } else { + igvn->replace_node(out_res, val); + } + } + } + + return new ConINode(TypeInt::ZERO); + } + + // If the Similar edge is no longer a load barrier, clear it + Node* similar = in(Similar); + if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { + set_req(Similar, phase->C->top()); + return this; + } + + if (can_reshape) { + // If this barrier is linked through the Similar edge by a + // dominated barrier and both barriers have the same Oop field, + // the dominated barrier can go away, so push it for reprocessing. + // We also want to avoid a barrier to depend on another dominating + // barrier through its Similar edge that itself depend on another + // barrier through its Similar edge and rather have the first + // depend on the third. + PhaseIterGVN* igvn = phase->is_IterGVN(); + Node* out_res = proj_out(Oop); + for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { + Node* u = out_res->fast_out(i); + if (u->is_LoadBarrier() && u->in(Similar) == out_res && + (u->in(Oop) == val || !u->in(Similar)->is_top())) { + igvn->_worklist.push(u); + } + } + + push_dominated_barriers(igvn); + } + + return NULL; +} + +void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) { + Node* out_res = proj_out_or_null(Oop); + if (out_res == NULL) { + return; + } + + for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { + Node* u = out_res->fast_out(i); + if (u->is_LoadBarrier() && u->in(Similar) == out_res) { + igvn->replace_input_of(u, Similar, igvn->C->top()); + --i; + --imax; + } + } +} + +bool LoadBarrierNode::has_true_uses() const { + Node* out_res = proj_out_or_null(Oop); + if (out_res == NULL) { + return false; + } + + for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { + Node* u = out_res->fast_out(i); + if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { + return true; + } + } + + return false; +} + +// == Accesses == + +Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const { + assert(!UseCompressedOops, "Not allowed"); + CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access(); + PhaseGVN& gvn = access.kit()->gvn(); + Compile* C = Compile::current(); + GraphKit* kit = access.kit(); + + Node* in_ctrl = cas->in(MemNode::Control); + Node* in_mem = cas->in(MemNode::Memory); + Node* in_adr = cas->in(MemNode::Address); + Node* in_val = cas->in(MemNode::ValueIn); + Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn); + + float likely = PROB_LIKELY(0.999); + + const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr(); + Compile::AliasType* alias_type = C->alias_type(adr_type); + int alias_idx = C->get_alias_index(adr_type); + + // Outer check - true: continue, false: load and check + Node* region = new RegionNode(3); + Node* phi = new PhiNode(region, TypeInt::BOOL); + Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type); + + // Inner check - is the healed ref equal to the expected + Node* region2 = new RegionNode(3); + Node* phi2 = new PhiNode(region2, TypeInt::BOOL); + Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type); + + // CAS node returns 0 or 1 + Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0))); + Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); + IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); + Node* then = gvn.transform(new IfTrueNode(iff)); + Node* elsen = gvn.transform(new IfFalseNode(iff)); + + Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas)); + + kit->set_memory(scmemproj1, alias_idx); + phi_mem->init_req(1, scmemproj1); + phi_mem2->init_req(2, scmemproj1); + + // CAS fail - reload and heal oop + Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); + Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); + Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); + Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); + + // Check load + Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); + Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); + Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); + Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); + IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); + Node* then2 = gvn.transform(new IfTrueNode(iff2)); + Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); + + // redo CAS + Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order())); + Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2)); + kit->set_control(elsen2); + kit->set_memory(scmemproj2, alias_idx); + + // Merge inner flow - check if healed oop was equal too expected. + region2->set_req(1, kit->control()); + region2->set_req(2, then2); + phi2->set_req(1, cas2); + phi2->set_req(2, kit->intcon(0)); + phi_mem2->init_req(1, scmemproj2); + kit->set_memory(phi_mem2, alias_idx); + + // Merge outer flow - then check if first cas succeded + region->set_req(1, then); + region->set_req(2, region2); + phi->set_req(1, kit->intcon(1)); + phi->set_req(2, phi2); + phi_mem->init_req(2, phi_mem2); + kit->set_memory(phi_mem, alias_idx); + + gvn.transform(region2); + gvn.transform(phi2); + gvn.transform(phi_mem2); + gvn.transform(region); + gvn.transform(phi); + gvn.transform(phi_mem); + + kit->set_control(region); + kit->insert_mem_bar(Op_MemBarCPUOrder); + + return phi; +} + +Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const { + CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access(); + GraphKit* kit = access.kit(); + PhaseGVN& gvn = kit->gvn(); + Compile* C = Compile::current(); + + Node* in_ctrl = cmpx->in(MemNode::Control); + Node* in_mem = cmpx->in(MemNode::Memory); + Node* in_adr = cmpx->in(MemNode::Address); + Node* in_val = cmpx->in(MemNode::ValueIn); + Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn); + + float likely = PROB_LIKELY(0.999); + + const TypePtr *adr_type = cmpx->get_ptr_type(); + Compile::AliasType* alias_type = C->alias_type(adr_type); + int alias_idx = C->get_alias_index(adr_type); + + // Outer check - true: continue, false: load and check + Node* region = new RegionNode(3); + Node* phi = new PhiNode(region, adr_type); + + // Inner check - is the healed ref equal to the expected + Node* region2 = new RegionNode(3); + Node* phi2 = new PhiNode(region2, adr_type); + + // Check if cmpx succeded + Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected)); + Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool(); + IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); + Node* then = gvn.transform(new IfTrueNode(iff)); + Node* elsen = gvn.transform(new IfFalseNode(iff)); + + Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx)); + kit->set_memory(scmemproj1, alias_idx); + + // CAS fail - reload and heal oop + Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); + Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); + Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); + Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); + + // Check load + Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); + Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); + Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); + Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); + IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); + Node* then2 = gvn.transform(new IfTrueNode(iff2)); + Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); + + // Redo CAS + Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order())); + Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2)); + kit->set_control(elsen2); + kit->set_memory(scmemproj2, alias_idx); + + // Merge inner flow - check if healed oop was equal too expected. + region2->set_req(1, kit->control()); + region2->set_req(2, then2); + phi2->set_req(1, cmpx2); + phi2->set_req(2, barrierdata); + + // Merge outer flow - then check if first cas succeded + region->set_req(1, then); + region->set_req(2, region2); + phi->set_req(1, cmpx); + phi->set_req(2, phi2); + + gvn.transform(region2); + gvn.transform(phi2); + gvn.transform(region); + gvn.transform(phi); + + kit->set_control(region); + kit->set_memory(in_mem, alias_idx); + kit->insert_mem_bar(Op_MemBarCPUOrder); + + return phi; +} + +Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const { + PhaseGVN& gvn = kit->gvn(); + Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed); + Node* transformed_barrier = gvn.transform(barrier); + + if (transformed_barrier->is_LoadBarrier()) { + if (barrier == transformed_barrier) { + kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control))); + } + return gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop)); + } else { + return val; + } +} + +static bool barrier_needed(C2Access access) { + return ZBarrierSet::barrier_needed(access.decorators(), access.type()); +} + +Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { + Node* p = BarrierSetC2::load_at_resolved(access, val_type); + if (!barrier_needed(access)) { + return p; + } + + bool conc_root = (access.decorators() & IN_CONCURRENT_ROOT) != 0; + bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; + + GraphKit* kit = access.kit(); + PhaseGVN& gvn = kit->gvn(); + Node* adr = access.addr().node(); + Node* heap_base_oop = access.base(); + bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0; + if (unsafe) { + if (!ZVerifyLoadBarriers) { + p = load_barrier(kit, p, adr); + } else { + if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) { + p = load_barrier(kit, p, adr); + } else { + IdealKit ideal(kit); + IdealVariable res(ideal); +#define __ ideal. + __ declarations_done(); + __ set(res, p); + __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); { + kit->sync_kit(ideal); + p = load_barrier(kit, p, adr); + __ set(res, p); + __ sync_kit(kit); + } __ end_if(); + kit->final_sync(ideal); + p = __ value(res); +#undef __ + } + } + return p; + } else { + return load_barrier(access.kit(), p, access.addr().node(), weak, true, true); + } +} + +Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* val_type) const { + Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); + if (!barrier_needed(access)) { + return result; + } + + access.set_needs_pinning(false); + return make_cmpx_loadbarrier(access); +} + +Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, + Node* new_val, const Type* value_type) const { + Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); + if (!barrier_needed(access)) { + return result; + } + + Node* load_store = access.raw_access(); + bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0; + bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR); + + if (!expected_is_null) { + if (weak_cas) { + access.set_needs_pinning(false); + load_store = make_cas_loadbarrier(access); + } else { + access.set_needs_pinning(false); + load_store = make_cas_loadbarrier(access); + } + } + + return load_store; +} + +Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { + Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); + if (!barrier_needed(access)) { + return result; + } + + Node* load_store = access.raw_access(); + Node* adr = access.addr().node(); + + return load_barrier(access.kit(), load_store, adr, false, false, false); +} + +// == Macro Expansion == + +void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { + Node* in_ctrl = barrier->in(LoadBarrierNode::Control); + Node* in_mem = barrier->in(LoadBarrierNode::Memory); + Node* in_val = barrier->in(LoadBarrierNode::Oop); + Node* in_adr = barrier->in(LoadBarrierNode::Address); + + Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); + Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); + + PhaseIterGVN &igvn = phase->igvn(); + + if (ZVerifyLoadBarriers) { + igvn.replace_node(out_res, in_val); + igvn.replace_node(out_ctrl, in_ctrl); + return; + } + + if (barrier->can_be_eliminated()) { + // Clone and pin the load for this barrier below the dominating + // barrier: the load cannot be allowed to float above the + // dominating barrier + Node* load = in_val; + Node* decode = NULL; + if (load->is_DecodeN()) { + decode = load; + load = load->in(1); + } + if (load->is_Load()) { + Node* new_load = load->clone(); + Node* addp = new_load->in(MemNode::Address); + assert(addp->is_AddP() || addp->is_Phi(), "bad address"); + Node* cast = new CastPPNode(addp, igvn.type(addp), true); + Node* ctrl = NULL; + Node* similar = barrier->in(LoadBarrierNode::Similar); + if (similar->is_Phi()) { + // already expanded + ctrl = similar->in(0); + } else { + assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape"); + ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control); + } + assert(ctrl != NULL, "bad control"); + cast->set_req(0, ctrl); + igvn.transform(cast); + new_load->set_req(MemNode::Address, cast); + igvn.transform(new_load); + + Node* new_in_val = new_load; + if (decode != NULL) { + new_in_val = decode->clone(); + new_in_val->set_req(1, new_load); + igvn.transform(new_in_val); + } + + igvn.replace_node(out_res, new_in_val); + igvn.replace_node(out_ctrl, in_ctrl); + return; + } + // cannot eliminate + } + + // There are two cases that require the basic loadbarrier + // 1) When the writeback of a healed oop must be avoided (swap) + // 2) When we must guarantee that no reload of is done (swap, cas, cmpx) + if (!barrier->is_writeback()) { + assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop"); + } + + if (!barrier->oop_reload_allowed()) { + expand_loadbarrier_basic(phase, barrier); + } else { + expand_loadbarrier_optimized(phase, barrier); + } +} + +// Basic loadbarrier using conventional arg passing +void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { + PhaseIterGVN &igvn = phase->igvn(); + + Node* in_ctrl = barrier->in(LoadBarrierNode::Control); + Node* in_mem = barrier->in(LoadBarrierNode::Memory); + Node* in_val = barrier->in(LoadBarrierNode::Oop); + Node* in_adr = barrier->in(LoadBarrierNode::Address); + + Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); + Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); + + float unlikely = PROB_UNLIKELY(0.999); + const Type* in_val_maybe_null_t = igvn.type(in_val); + + Node* jthread = igvn.transform(new ThreadLocalNode()); + Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); + Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); + Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); + Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); + Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); + Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); + IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); + Node* then = igvn.transform(new IfTrueNode(iff)); + Node* elsen = igvn.transform(new IfFalseNode(iff)); + + Node* result_region; + Node* result_val; + + result_region = new RegionNode(3); + result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM); + + result_region->set_req(1, elsen); + Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t)); + res->init_req(0, elsen); + result_val->set_req(1, res); + + const TypeFunc *tf = load_barrier_Type(); + Node* call; + if (barrier->is_weak()) { + call = new CallLeafNode(tf, + ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(), + "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded", + TypeRawPtr::BOTTOM); + } else { + call = new CallLeafNode(tf, + ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), + "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded", + TypeRawPtr::BOTTOM); + } + + call->init_req(TypeFunc::Control, then); + call->init_req(TypeFunc::I_O , phase->top()); + call->init_req(TypeFunc::Memory , in_mem); + call->init_req(TypeFunc::FramePtr, phase->top()); + call->init_req(TypeFunc::ReturnAdr, phase->top()); + call->init_req(TypeFunc::Parms+0, in_val); + if (barrier->is_writeback()) { + call->init_req(TypeFunc::Parms+1, in_adr); + } else { + // when slow path is called with a null adr, the healed oop will not be written back + call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT)); + } + call = igvn.transform(call); + + Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control)); + res = igvn.transform(new ProjNode(call, TypeFunc::Parms)); + res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t)); + + result_region->set_req(2, ctrl); + result_val->set_req(2, res); + + result_region = igvn.transform(result_region); + result_val = igvn.transform(result_val); + + if (out_ctrl != NULL) { // added if cond + igvn.replace_node(out_ctrl, result_region); + } + igvn.replace_node(out_res, result_val); +} + +// Optimized, low spill, loadbarrier variant using stub specialized on register used +void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { + PhaseIterGVN &igvn = phase->igvn(); +#ifdef PRINT_NODE_TRAVERSALS + Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop); +#endif + + Node* in_ctrl = barrier->in(LoadBarrierNode::Control); + Node* in_mem = barrier->in(LoadBarrierNode::Memory); + Node* in_val = barrier->in(LoadBarrierNode::Oop); + Node* in_adr = barrier->in(LoadBarrierNode::Address); + + Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); + Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); + + assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); + +#ifdef PRINT_NODE_TRAVERSALS + tty->print("\n\n\nBefore barrier optimization:\n"); + traverse(barrier, out_ctrl, out_res, -1); + + tty->print("\nBefore barrier optimization: preceding_barrier_node\n"); + traverse(preceding_barrier_node, out_ctrl, out_res, -1); +#endif + + float unlikely = PROB_UNLIKELY(0.999); + + Node* jthread = igvn.transform(new ThreadLocalNode()); + Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); + Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, + TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), + MemNode::unordered)); + Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); + Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); + Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); + Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); + IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); + Node* then = igvn.transform(new IfTrueNode(iff)); + Node* elsen = igvn.transform(new IfFalseNode(iff)); + + Node* slow_path_surrogate; + if (!barrier->is_weak()) { + slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), + (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); + } else { + slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), + (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); + } + + Node *new_loadp; + new_loadp = slow_path_surrogate; + // create the final region/phi pair to converge cntl/data paths to downstream code + Node* result_region = igvn.transform(new RegionNode(3)); + result_region->set_req(1, then); + result_region->set_req(2, elsen); + + Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); + result_phi->set_req(1, new_loadp); + result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); + + // finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution + // igvn.replace_node(out_ctrl, result_region); + if (out_ctrl != NULL) { // added if cond + igvn.replace_node(out_ctrl, result_region); + } + igvn.replace_node(out_res, result_phi); + + assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); + +#ifdef PRINT_NODE_TRAVERSALS + tty->print("\nAfter barrier optimization: old out_ctrl\n"); + traverse(out_ctrl, out_ctrl, out_res, -1); + tty->print("\nAfter barrier optimization: old out_res\n"); + traverse(out_res, out_ctrl, out_res, -1); + tty->print("\nAfter barrier optimization: old barrier\n"); + traverse(barrier, out_ctrl, out_res, -1); + tty->print("\nAfter barrier optimization: preceding_barrier_node\n"); + traverse(preceding_barrier_node, result_region, result_phi, -1); +#endif + + return; +} + +bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { + Compile* C = Compile::current(); + PhaseIterGVN &igvn = macro->igvn(); + ZBarrierSetC2State* s = state(); + if (s->load_barrier_count() > 0) { +#ifdef ASSERT + verify_gc_barriers(false); +#endif + igvn.set_delay_transform(true); + int skipped = 0; + while (s->load_barrier_count() > skipped) { + int load_barrier_count = s->load_barrier_count(); + LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); + if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { + // node is unreachable, so don't try to expand it + s->remove_load_barrier_node(n); + continue; + } + if (!n->can_be_eliminated()) { + skipped++; + continue; + } + expand_loadbarrier_node(macro, n); + assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); + if (C->failing()) return true; + } + while (s->load_barrier_count() > 0) { + int load_barrier_count = s->load_barrier_count(); + LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1); + assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already"); + assert(!n->can_be_eliminated(), "should have been processed already"); + expand_loadbarrier_node(macro, n); + assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); + if (C->failing()) return true; + } + igvn.set_delay_transform(false); + igvn.optimize(); + if (C->failing()) return true; + } + return false; +} + +// == Loop optimization == + +static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { + PhaseIterGVN &igvn = phase->igvn(); + Compile* C = Compile::current(); + + LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round); + if (lb2 != NULL) { + if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) { + assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), ""); + igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop)); + C->set_major_progress(); + } else { + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + + Node* val = lb->proj_out(LoadBarrierNode::Oop); + assert(lb2->has_true_uses(), ""); + assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), ""); + + phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); + phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); + igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop)); + + return true; + } + } + return false; +} + +static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) { + assert(dom->is_Region() || i == -1, ""); + Node* m = mem; + while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) { + if (m->is_Mem()) { + assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, ""); + m = m->in(MemNode::Memory); + } else if (m->is_MergeMem()) { + m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw); + } else if (m->is_Phi()) { + if (m->in(0) == dom && i != -1) { + m = m->in(i); + break; + } else { + m = m->in(LoopNode::EntryControl); + } + } else if (m->is_Proj()) { + m = m->in(0); + } else if (m->is_SafePoint() || m->is_MemBar()) { + m = m->in(TypeFunc::Memory); + } else { +#ifdef ASSERT + m->dump(); +#endif + ShouldNotReachHere(); + } + } + return m; +} + +static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) { + PhaseIterGVN &igvn = phase->igvn(); + Compile* C = Compile::current(); + Node* the_clone = lb->clone(); + the_clone->set_req(LoadBarrierNode::Control, ctl); + the_clone->set_req(LoadBarrierNode::Memory, mem); + if (oop_in != NULL) { + the_clone->set_req(LoadBarrierNode::Oop, oop_in); + } + + LoadBarrierNode* new_lb = the_clone->as_LoadBarrier(); + igvn.register_new_node_with_optimizer(new_lb); + IdealLoopTree *loop = phase->get_loop(new_lb->in(0)); + phase->set_ctrl(new_lb, new_lb->in(0)); + phase->set_loop(new_lb, loop); + phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1); + if (!loop->_child) { + loop->_body.push(new_lb); + } + + Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control); + igvn.register_new_node_with_optimizer(proj_ctl); + phase->set_ctrl(proj_ctl, proj_ctl->in(0)); + phase->set_loop(proj_ctl, loop); + phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1); + if (!loop->_child) { + loop->_body.push(proj_ctl); + } + + Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop); + phase->register_new_node(proj_oop, new_lb); + + if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) { + LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier(); + if (!phase->is_dominator(similar, ctl)) { + igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top()); + } + } + + return new_lb; +} + +static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) { + PhaseIterGVN &igvn = phase->igvn(); + Node* val = lb->proj_out(LoadBarrierNode::Oop); + igvn.replace_node(val, new_val); + phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); + phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); +} + +static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) { + PhaseIterGVN &igvn = phase->igvn(); + Compile* C = Compile::current(); + + if (lb->in(LoadBarrierNode::Oop)->is_Phi()) { + Node* oop_phi = lb->in(LoadBarrierNode::Oop); + + if (oop_phi->req() == 2) { + // Ignore phis with only one input + return false; + } + + if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)), + oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) { + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + + RegionNode* region = oop_phi->in(0)->as_Region(); + + int backedge = LoopNode::LoopBackControl; + if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) { + Node* c = region->in(backedge)->in(0)->in(0); + assert(c->unique_ctrl_out() == region->in(backedge)->in(0), ""); + Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge); + Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop; + if (!phase->is_dominator(oop_c, c)) { + return false; + } + } + + // If the node on the backedge above the phi is the node itself - we have a self loop. + // Don't clone - this will be folded later. + if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) { + return false; + } + + bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined(); + Node *phi = oop_phi->clone(); + + for (uint i = 1; i < region->req(); i++) { + Node* ctrl = region->in(i); + if (ctrl != C->top()) { + assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), ""); + + Node* mem = lb->in(LoadBarrierNode::Memory); + Node* m = find_dominating_memory(phase, mem, region, i); + + if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) { + ctrl = ctrl->in(0)->in(0); + } else if (region->is_Loop() && is_strip_mined) { + // If this is a strip mined loop, control must move above OuterStripMinedLoop + assert(i == LoopNode::EntryControl, "check"); + assert(ctrl->is_OuterStripMinedLoop(), "sanity"); + ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl); + } + + LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i)); + Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control); + + if (is_strip_mined && (i == LoopNode::EntryControl)) { + assert(region->in(i)->is_OuterStripMinedLoop(), ""); + igvn.replace_input_of(region->in(i), i, out_ctrl); + } else if (ctrl == region->in(i)) { + igvn.replace_input_of(region, i, out_ctrl); + } else { + Node* iff = region->in(i)->in(0); + igvn.replace_input_of(iff, 0, out_ctrl); + phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1); + } + phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop)); + } + } + phase->register_new_node(phi, region); + replace_barrier(phase, lb, phi); + + if (region->is_Loop()) { + // Load barrier moved to the back edge of the Loop may now + // have a safepoint on the path to the barrier on the Similar + // edge + igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top()); + Node* head = region->in(LoopNode::EntryControl); + phase->set_idom(region, head, phase->dom_depth(head)+1); + phase->recompute_dom_depth(); + if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) { + head->as_CountedLoop()->set_normal_loop(); + } + } + + return true; + } + } + + return false; +} + +static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) { + PhaseIterGVN &igvn = phase->igvn(); + IdealLoopTree *lb_loop = phase->get_loop(lb->in(0)); + if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) { + Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop)); + IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl); + IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address))); + if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) { + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + + Node* head = lb_loop->_head; + assert(head->is_Loop(), ""); + + if (phase->is_dominator(head, oop_ctrl)) { + assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, ""); + assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), ""); + return false; + } + + if (head->is_CountedLoop()) { + CountedLoopNode* cloop = head->as_CountedLoop(); + if (cloop->is_main_loop()) { + cloop->set_normal_loop(); + } + // When we are moving barrier out of a counted loop, + // make sure we move it all the way out of the strip mined outer loop. + if (cloop->is_strip_mined()) { + head = cloop->outer_loop(); + } + } + + Node* mem = lb->in(LoadBarrierNode::Memory); + Node* m = find_dominating_memory(phase, mem, head, -1); + + LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL); + + assert(phase->idom(head) == head->in(LoopNode::EntryControl), ""); + Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); + igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl); + phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1); + + replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop)); + + phase->recompute_dom_depth(); + + return true; + } + } + + return false; +} + +static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) { + PhaseIterGVN &igvn = phase->igvn(); + Node* in_val = lb->in(LoadBarrierNode::Oop); + for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) { + Node* u = in_val->fast_out(i); + if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { + Node* this_ctrl = lb->in(LoadBarrierNode::Control); + Node* other_ctrl = u->in(LoadBarrierNode::Control); + + Node* lca = phase->dom_lca(this_ctrl, other_ctrl); + bool ok = true; + + Node* proj1 = NULL; + Node* proj2 = NULL; + + while (this_ctrl != lca && ok) { + if (this_ctrl->in(0) != NULL && + this_ctrl->in(0)->is_MultiBranch()) { + if (this_ctrl->in(0)->in(0) == lca) { + assert(proj1 == NULL, ""); + assert(this_ctrl->is_Proj(), ""); + proj1 = this_ctrl; + } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { + ok = false; + } + } + this_ctrl = phase->idom(this_ctrl); + } + while (other_ctrl != lca && ok) { + if (other_ctrl->in(0) != NULL && + other_ctrl->in(0)->is_MultiBranch()) { + if (other_ctrl->in(0)->in(0) == lca) { + assert(other_ctrl->is_Proj(), ""); + assert(proj2 == NULL, ""); + proj2 = other_ctrl; + } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { + ok = false; + } + } + other_ctrl = phase->idom(other_ctrl); + } + assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), ""); + if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) { + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + u->as_LoadBarrier()->fix_similar_in_uses(&igvn); + + Node* split = lca->unique_ctrl_out(); + assert(split->in(0) == lca, ""); + + Node* mem = lb->in(LoadBarrierNode::Memory); + Node* m = find_dominating_memory(phase, mem, split, -1); + LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL); + + Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); + igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control)); + phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1); + + Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop); + replace_barrier(phase, lb, proj_oop); + replace_barrier(phase, u->as_LoadBarrier(), proj_oop); + + phase->recompute_dom_depth(); + + return true; + } + } + } + + return false; +} + +static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { + Compile* C = Compile::current(); + + if (!C->directive()->ZOptimizeLoadBarriersOption) { + return; + } + + if (lb->has_true_uses()) { + if (replace_with_dominating_barrier(phase, lb, last_round)) { + return; + } + + if (split_barrier_thru_phi(phase, lb)) { + return; + } + + if (move_out_of_loop(phase, lb)) { + return; + } + + if (common_barriers(phase, lb)) { + return; + } + } +} + +void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) { + if (node->is_LoadBarrier()) { + optimize_load_barrier(phase, node->as_LoadBarrier(), last_round); + } +} + +// == Verification == + +#ifdef ASSERT + +static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) { + if (visited.test_set(n->_idx)) { + return true; + } + + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + Node* u = n->fast_out(i); + if (u->is_LoadBarrier()) { + } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) { + if (!look_for_barrier(u, post_parse, visited)) { + return false; + } + } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) { + if (!look_for_barrier(u, post_parse, visited)) { + return false; + } + } else if (u->Opcode() != Op_SCMemProj) { + tty->print("bad use"); u->dump(); + return false; + } + } + + return true; +} + +void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { + ZBarrierSetC2State* s = state(); + Compile* C = Compile::current(); + ResourceMark rm; + VectorSet visited(Thread::current()->resource_area()); + for (int i = 0; i < s->load_barrier_count(); i++) { + LoadBarrierNode* n = s->load_barrier_node(i); + + // The dominating barrier on the same address if it exists and + // this barrier must not be applied on the value from the same + // load otherwise the value is not reloaded before it's used the + // second time. + assert(n->in(LoadBarrierNode::Similar)->is_top() || + (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && + n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && + n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), + "broken similar edge"); + + assert(post_parse || n->as_LoadBarrier()->has_true_uses(), + "found unneeded load barrier"); + + // Several load barrier nodes chained through their Similar edge + // break the code that remove the barriers in final graph reshape. + assert(n->in(LoadBarrierNode::Similar)->is_top() || + (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && + n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()), + "chain of Similar load barriers"); + + if (!n->in(LoadBarrierNode::Similar)->is_top()) { + ResourceMark rm; + Unique_Node_List wq; + Node* other = n->in(LoadBarrierNode::Similar)->in(0); + wq.push(n); + bool ok = true; + bool dom_found = false; + for (uint next = 0; next < wq.size(); ++next) { + Node *n = wq.at(next); + assert(n->is_CFG(), ""); + assert(!n->is_SafePoint(), ""); + + if (n == other) { + continue; + } + + if (n->is_Region()) { + for (uint i = 1; i < n->req(); i++) { + Node* m = n->in(i); + if (m != NULL) { + wq.push(m); + } + } + } else { + Node* m = n->in(0); + if (m != NULL) { + wq.push(m); + } + } + } + } + + if (ZVerifyLoadBarriers) { + if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) { + visited.Clear(); + bool found = look_for_barrier(n, post_parse, visited); + if (!found) { + n->dump(1); + n->dump(-3); + stringStream ss; + C->method()->print_short_name(&ss); + tty->print_cr("-%s-", ss.as_string()); + assert(found, ""); + } + } + } + } +} + +#endif --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp 2018-06-01 22:30:16.054820991 +0200 @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_C2_ZBARRIERSETC2_HPP +#define SHARE_GC_Z_C2_ZBARRIERSETC2_HPP + +#include "gc/shared/c2/barrierSetC2.hpp" +#include "memory/allocation.hpp" +#include "opto/node.hpp" +#include "utilities/growableArray.hpp" + +class LoadBarrierNode : public MultiNode { +private: + bool _weak; + bool _writeback; // Controls if the barrier writes the healed oop back to memory + // A swap on a memory location must never write back the healed oop + bool _oop_reload_allowed; // Controls if the barrier are allowed to reload the oop from memory + // before healing, otherwise both the oop and the address must be passed to the + // barrier from the oop + + static bool is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n); + void push_dominated_barriers(PhaseIterGVN* igvn) const; + +public: + enum { + Control, + Memory, + Oop, + Address, + Number_of_Outputs = Address, + Similar, + Number_of_Inputs + }; + + LoadBarrierNode(Compile* C, + Node* c, + Node* mem, + Node* val, + Node* adr, + bool weak, + bool writeback, + bool oop_reload_allowed); + + virtual int Opcode() const; + virtual const Type *bottom_type() const; + virtual const Type *Value(PhaseGVN *phase) const; + virtual Node *Identity(PhaseGVN *phase); + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + + LoadBarrierNode* has_dominating_barrier(PhaseIdealLoop* phase, + bool linear_only, + bool look_for_similar); + + void fix_similar_in_uses(PhaseIterGVN* igvn); + + bool has_true_uses() const; + + bool can_be_eliminated() const { + return !in(Similar)->is_top(); + } + + bool is_weak() const { + return _weak; + } + + bool is_writeback() const { + return _writeback; + } + + bool oop_reload_allowed() const { + return _oop_reload_allowed; + } +}; + +class LoadBarrierSlowRegNode : public LoadPNode { +public: + LoadBarrierSlowRegNode(Node *c, + Node *mem, + Node *adr, + const TypePtr *at, + const TypePtr* t, + MemOrd mo, + ControlDependency control_dependency = DependsOnlyOnTest) + : LoadPNode(c, mem, adr, at, t, mo, control_dependency) {} + + virtual const char * name() { + return "LoadBarrierSlowRegNode"; + } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + return NULL; + } + + virtual int Opcode() const; +}; + +class LoadBarrierWeakSlowRegNode : public LoadPNode { +public: + LoadBarrierWeakSlowRegNode(Node *c, + Node *mem, + Node *adr, + const TypePtr *at, + const TypePtr* t, + MemOrd mo, + ControlDependency control_dependency = DependsOnlyOnTest) + : LoadPNode(c, mem, adr, at, t, mo, control_dependency) {} + + virtual const char * name() { + return "LoadBarrierWeakSlowRegNode"; + } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { + return NULL; + } + + virtual int Opcode() const; +}; + +class ZBarrierSetC2State : public ResourceObj { +private: + // List of load barrier nodes which need to be expanded before matching + GrowableArray* _load_barrier_nodes; + +public: + ZBarrierSetC2State(Arena* comp_arena); + int load_barrier_count() const; + void add_load_barrier_node(LoadBarrierNode* n); + void remove_load_barrier_node(LoadBarrierNode* n); + LoadBarrierNode* load_barrier_node(int idx) const; +}; + +class ZBarrierSetC2 : public BarrierSetC2 { +private: + ZBarrierSetC2State* state() const; + Node* make_cas_loadbarrier(C2AtomicAccess& access) const; + Node* make_cmpx_loadbarrier(C2AtomicAccess& access) const; + void expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const; + void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const; + void expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const; + const TypeFunc* load_barrier_Type() const; + +protected: + virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const; + virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, + Node* expected_val, + Node* new_val, + const Type* val_type) const; + virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, + Node* expected_val, + Node* new_val, + const Type* value_type) const; + virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, + Node* new_val, + const Type* val_type) const; + +public: + Node* load_barrier(GraphKit* kit, + Node* val, + Node* adr, + bool weak = false, + bool writeback = true, + bool oop_reload_allowed = true) const; + + virtual void* create_barrier_state(Arena* comp_arena) const; + virtual bool is_gc_barrier_node(Node* node) const; + virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } + virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const; + virtual void add_users_to_worklist(Unique_Node_List* worklist) const; + virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const; + virtual void register_potential_barrier_node(Node* node) const; + virtual void unregister_potential_barrier_node(Node* node) const; + virtual bool array_copy_requires_gc_barriers(BasicType type) const { return true; } + virtual Node* step_over_gc_barrier(Node* c) const { return c; } + // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be + // expanded later, then now is the time to do so. + virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const; + + static void find_dominating_barriers(PhaseIterGVN& igvn); + static void loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round); + +#ifdef ASSERT + virtual void verify_gc_barriers(bool post_parse) const; +#endif +}; + +#endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/vmStructs_z.cpp 2018-06-01 22:30:16.445837869 +0200 @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/z/vmStructs_z.hpp" + +ZGlobalsForVMStructs::ZGlobalsForVMStructs() : + _ZGlobalPhase(&ZGlobalPhase), + _ZAddressGoodMask(&ZAddressGoodMask), + _ZAddressBadMask(&ZAddressBadMask), + _ZAddressWeakBadMask(&ZAddressWeakBadMask), + _ZObjectAlignmentSmallShift(&ZObjectAlignmentSmallShift), + _ZObjectAlignmentSmall(&ZObjectAlignmentSmall) { +} + +ZGlobalsForVMStructs ZGlobalsForVMStructs::_instance; +ZGlobalsForVMStructs* ZGlobalsForVMStructs::_instance_p = &ZGlobalsForVMStructs::_instance; --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/vmStructs_z.hpp 2018-06-01 22:30:16.825854272 +0200 @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_Z_VMSTRUCTS_Z_HPP +#define SHARE_VM_GC_Z_VMSTRUCTS_Z_HPP + +#include "gc/z/zAddressRangeMap.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zHeap.hpp" +#include "gc/z/zPageAllocator.hpp" +#include "gc/z/zPhysicalMemory.hpp" +#include "utilities/macros.hpp" + +// Expose some ZGC globals to the SA agent. +class ZGlobalsForVMStructs { + static ZGlobalsForVMStructs _instance; + +public: + static ZGlobalsForVMStructs* _instance_p; + + ZGlobalsForVMStructs(); + + uint32_t* _ZGlobalPhase; + + uintptr_t* _ZAddressGoodMask; + uintptr_t* _ZAddressBadMask; + uintptr_t* _ZAddressWeakBadMask; + + const int* _ZObjectAlignmentSmallShift; + const int* _ZObjectAlignmentSmall; +}; + +typedef ZAddressRangeMap ZAddressRangeMapForPageTable; + +#define VM_STRUCTS_ZGC(nonstatic_field, volatile_nonstatic_field, static_field) \ + static_field(ZGlobalsForVMStructs, _instance_p, ZGlobalsForVMStructs*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZGlobalPhase, uint32_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZAddressGoodMask, uintptr_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZAddressBadMask, uintptr_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZAddressWeakBadMask, uintptr_t*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZObjectAlignmentSmallShift, const int*) \ + nonstatic_field(ZGlobalsForVMStructs, _ZObjectAlignmentSmall, const int*) \ + \ + nonstatic_field(ZCollectedHeap, _heap, ZHeap) \ + \ + nonstatic_field(ZHeap, _page_allocator, ZPageAllocator) \ + nonstatic_field(ZHeap, _pagetable, ZPageTable) \ + \ + nonstatic_field(ZPage, _type, const uint8_t) \ + nonstatic_field(ZPage, _virtual, const ZVirtualMemory) \ + nonstatic_field(ZPage, _forwarding, ZForwardingTable) \ + \ + nonstatic_field(ZPageAllocator, _physical, ZPhysicalMemoryManager) \ + nonstatic_field(ZPageAllocator, _used, size_t) \ + \ + nonstatic_field(ZPageTable, _map, ZAddressRangeMapForPageTable) \ + \ + nonstatic_field(ZAddressRangeMapForPageTable, _map, ZPageTableEntry* const) \ + \ + nonstatic_field(ZVirtualMemory, _start, uintptr_t) \ + nonstatic_field(ZVirtualMemory, _end, uintptr_t) \ + \ + nonstatic_field(ZForwardingTable, _table, ZForwardingTableEntry*) \ + nonstatic_field(ZForwardingTable, _size, size_t) \ + \ + nonstatic_field(ZPhysicalMemoryManager, _max_capacity, const size_t) \ + nonstatic_field(ZPhysicalMemoryManager, _capacity, size_t) + +#define VM_INT_CONSTANTS_ZGC(declare_constant, declare_constant_with_value) \ + declare_constant(ZPhaseRelocate) \ + declare_constant(ZPageTypeSmall) \ + declare_constant(ZPageTypeMedium) \ + declare_constant(ZPageTypeLarge) \ + declare_constant(ZObjectAlignmentMediumShift) \ + declare_constant(ZObjectAlignmentLargeShift) + +#define VM_LONG_CONSTANTS_ZGC(declare_constant) \ + declare_constant(ZPageSizeSmallShift) \ + declare_constant(ZPageSizeMediumShift) \ + declare_constant(ZPageSizeMinShift) \ + declare_constant(ZAddressOffsetShift) \ + declare_constant(ZAddressOffsetBits) \ + declare_constant(ZAddressOffsetMask) \ + declare_constant(ZAddressSpaceStart) + +#define VM_TYPES_ZGC(declare_type, declare_toplevel_type, declare_integer_type) \ + declare_toplevel_type(ZGlobalsForVMStructs) \ + declare_type(ZCollectedHeap, CollectedHeap) \ + declare_toplevel_type(ZHeap) \ + declare_toplevel_type(ZPage) \ + declare_toplevel_type(ZPageAllocator) \ + declare_toplevel_type(ZPageTable) \ + declare_toplevel_type(ZPageTableEntry) \ + declare_toplevel_type(ZAddressRangeMapForPageTable) \ + declare_toplevel_type(ZVirtualMemory) \ + declare_toplevel_type(ZForwardingTable) \ + declare_toplevel_type(ZForwardingTableEntry) \ + declare_toplevel_type(ZPhysicalMemoryManager) + +#endif // SHARE_VM_GC_Z_VMSTRUCTS_Z_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zAddress.cpp 2018-06-01 22:30:17.195870244 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zGlobals.hpp" +#include "runtime/thread.hpp" + +void ZAddressMasks::set_good_mask(uintptr_t mask) { + uintptr_t old_bad_mask = ZAddressBadMask; + ZAddressGoodMask = mask; + ZAddressBadMask = ZAddressGoodMask ^ ZAddressMetadataMask; + ZAddressWeakBadMask = (ZAddressGoodMask | ZAddressMetadataRemapped | ZAddressMetadataFinalizable) ^ ZAddressMetadataMask; +} + +void ZAddressMasks::initialize() { + ZAddressMetadataMarked = ZAddressMetadataMarked0; + set_good_mask(ZAddressMetadataRemapped); +} + +void ZAddressMasks::flip_to_marked() { + ZAddressMetadataMarked ^= (ZAddressMetadataMarked0 | ZAddressMetadataMarked1); + set_good_mask(ZAddressMetadataMarked); +} + +void ZAddressMasks::flip_to_remapped() { + set_good_mask(ZAddressMetadataRemapped); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zAddress.hpp 2018-06-01 22:30:17.574886604 +0200 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZADDRESS_HPP +#define SHARE_GC_Z_ZADDRESS_HPP + +#include "memory/allocation.hpp" + +class ZAddress : public AllStatic { +public: + static bool is_null(uintptr_t value); + static bool is_bad(uintptr_t value); + static bool is_good(uintptr_t value); + static bool is_good_or_null(uintptr_t value); + static bool is_weak_bad(uintptr_t value); + static bool is_weak_good(uintptr_t value); + static bool is_weak_good_or_null(uintptr_t value); + static bool is_marked(uintptr_t value); + static bool is_finalizable(uintptr_t value); + static bool is_remapped(uintptr_t value); + + static uintptr_t address(uintptr_t value); + static uintptr_t offset(uintptr_t value); + static uintptr_t good(uintptr_t value); + static uintptr_t good_or_null(uintptr_t value); + static uintptr_t finalizable_good(uintptr_t value); + static uintptr_t marked(uintptr_t value); + static uintptr_t marked0(uintptr_t value); + static uintptr_t marked1(uintptr_t value); + static uintptr_t remapped(uintptr_t value); + static uintptr_t remapped_or_null(uintptr_t value); +}; + +class ZAddressMasks : public AllStatic { + friend class ZAddressTest; + +private: + static void set_good_mask(uintptr_t mask); + +public: + static void initialize(); + static void flip_to_marked(); + static void flip_to_remapped(); +}; + +#endif // SHARE_GC_Z_ZADDRESS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zAddress.inline.hpp 2018-06-01 22:30:17.957903136 +0200 @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZADDRESS_INLINE_HPP +#define SHARE_GC_Z_ZADDRESS_INLINE_HPP + +#include "gc/z/zAddress.hpp" +#include "gc/z/zGlobals.hpp" +#include "utilities/macros.hpp" +#include OS_CPU_HEADER_INLINE(gc/z/zAddress) + +inline bool ZAddress::is_null(uintptr_t value) { + return value == 0; +} + +inline bool ZAddress::is_bad(uintptr_t value) { + return value & ZAddressBadMask; +} + +inline bool ZAddress::is_good(uintptr_t value) { + return !is_bad(value) && !is_null(value); +} + +inline bool ZAddress::is_good_or_null(uintptr_t value) { + // Checking if an address is "not bad" is an optimized version of + // checking if it's "good or null", which eliminates an explicit + // null check. However, the implicit null check only checks that + // the mask bits are zero, not that the entire address is zero. + // This means that an address without mask bits would pass through + // the barrier as if it was null. This should be harmless as such + // addresses should ever be passed through the barrier. + const bool result = !is_bad(value); + assert((is_good(value) || is_null(value)) == result, "Bad address"); + return result; +} + +inline bool ZAddress::is_weak_bad(uintptr_t value) { + return value & ZAddressWeakBadMask; +} + +inline bool ZAddress::is_weak_good(uintptr_t value) { + return !is_weak_bad(value) && !is_null(value); +} + +inline bool ZAddress::is_weak_good_or_null(uintptr_t value) { + return !is_weak_bad(value); +} + +inline bool ZAddress::is_marked(uintptr_t value) { + return value & ZAddressMetadataMarked; +} + +inline bool ZAddress::is_finalizable(uintptr_t value) { + return value & ZAddressMetadataFinalizable; +} + +inline bool ZAddress::is_remapped(uintptr_t value) { + return value & ZAddressMetadataRemapped; +} + +inline uintptr_t ZAddress::offset(uintptr_t value) { + return value & ZAddressOffsetMask; +} + +inline uintptr_t ZAddress::good(uintptr_t value) { + return address(offset(value) | ZAddressGoodMask); +} + +inline uintptr_t ZAddress::good_or_null(uintptr_t value) { + return is_null(value) ? 0 : good(value); +} + +inline uintptr_t ZAddress::finalizable_good(uintptr_t value) { + return address(offset(value) | ZAddressMetadataFinalizable | ZAddressGoodMask); +} + +inline uintptr_t ZAddress::marked(uintptr_t value) { + return address(offset(value) | ZAddressMetadataMarked); +} + +inline uintptr_t ZAddress::marked0(uintptr_t value) { + return address(offset(value) | ZAddressMetadataMarked0); +} + +inline uintptr_t ZAddress::marked1(uintptr_t value) { + return address(offset(value) | ZAddressMetadataMarked1); +} + +inline uintptr_t ZAddress::remapped(uintptr_t value) { + return address(offset(value) | ZAddressMetadataRemapped); +} + +inline uintptr_t ZAddress::remapped_or_null(uintptr_t value) { + return is_null(value) ? 0 : remapped(value); +} + +#endif // SHARE_GC_Z_ZADDRESS_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zAddressRangeMap.hpp 2018-06-01 22:30:18.335919453 +0200 @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZADDRESSRANGEMAP_HPP +#define SHARE_GC_Z_ZADDRESSRANGEMAP_HPP + +#include "memory/allocation.hpp" + +template +class ZAddressRangeMapIterator; + +template +class ZAddressRangeMap { + friend class VMStructs; + friend class ZAddressRangeMapIterator; + +private: + T* const _map; + + size_t index_for_addr(uintptr_t addr) const; + size_t size() const; + +public: + ZAddressRangeMap(); + ~ZAddressRangeMap(); + + T get(uintptr_t addr) const; + void put(uintptr_t addr, T value); +}; + +template +class ZAddressRangeMapIterator : public StackObj { +public: + const ZAddressRangeMap* const _map; + size_t _next; + +public: + ZAddressRangeMapIterator(const ZAddressRangeMap* map); + + bool next(T* value); +}; + +#endif // SHARE_GC_Z_ZADDRESSRANGEMAP_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zAddressRangeMap.inline.hpp 2018-06-01 22:30:18.710935640 +0200 @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZADDRESSRANGEMAP_INLINE_HPP +#define SHARE_GC_Z_ZADDRESSRANGEMAP_INLINE_HPP + +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zAddressRangeMap.hpp" +#include "gc/z/zGlobals.hpp" +#include "memory/allocation.inline.hpp" + +template +ZAddressRangeMap::ZAddressRangeMap() : + _map(MmapArrayAllocator::allocate(size(), mtGC)) {} + +template +ZAddressRangeMap::~ZAddressRangeMap() { + MmapArrayAllocator::free(_map, size()); +} + +template +size_t ZAddressRangeMap::index_for_addr(uintptr_t addr) const { + assert(!ZAddress::is_null(addr), "Invalid address"); + + const size_t index = ZAddress::offset(addr) >> AddressRangeShift; + assert(index < size(), "Invalid index"); + + return index; +} + +template +size_t ZAddressRangeMap::size() const { + return ZAddressOffsetMax >> AddressRangeShift; +} + +template +T ZAddressRangeMap::get(uintptr_t addr) const { + const uintptr_t index = index_for_addr(addr); + return _map[index]; +} + +template +void ZAddressRangeMap::put(uintptr_t addr, T value) { + const uintptr_t index = index_for_addr(addr); + _map[index] = value; +} + +template +inline ZAddressRangeMapIterator::ZAddressRangeMapIterator(const ZAddressRangeMap* map) : + _map(map), + _next(0) {} + +template +inline bool ZAddressRangeMapIterator::next(T* value) { + if (_next < _map->size()) { + *value = _map->_map[_next++]; + return true; + } + + // End of map + return false; +} + +#endif // SHARE_GC_Z_ZADDRESSRANGEMAP_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zAllocationFlags.hpp 2018-06-01 22:30:19.078951525 +0200 @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZALLOCATIONFLAGS_HPP +#define SHARE_GC_Z_ZALLOCATIONFLAGS_HPP + +#include "gc/z/zBitField.hpp" +#include "memory/allocation.hpp" + +// +// Allocation flags layout +// ----------------------- +// +// 7 4 3 2 1 0 +// +---+-+-+-+-+-+ +// |000|1|1|1|1|1| +// +---+-+-+-+-+-+ +// | | | | | | +// | | | | | * 0-0 Java Thread Flag (1-bit) +// | | | | | +// | | | | * 1-1 Worker Thread Flag (1-bit) +// | | | | +// | | | * 2-2 Non-Blocking Flag (1-bit) +// | | | +// | | * 3-3 Relocation Flag (1-bit) +// | | +// | * 4-4 No Reserve Flag (1-bit) +// | +// * 7-5 Unused (3-bits) +// + +class ZAllocationFlags { +private: + typedef ZBitField field_java_thread; + typedef ZBitField field_worker_thread; + typedef ZBitField field_non_blocking; + typedef ZBitField field_relocation; + typedef ZBitField field_no_reserve; + + uint8_t _flags; + +public: + ZAllocationFlags() : + _flags(0) {} + + void set_java_thread() { + _flags |= field_java_thread::encode(true); + } + + void set_worker_thread() { + _flags |= field_worker_thread::encode(true); + } + + void set_non_blocking() { + _flags |= field_non_blocking::encode(true); + } + + void set_relocation() { + _flags |= field_relocation::encode(true); + } + + void set_no_reserve() { + _flags |= field_no_reserve::encode(true); + } + + bool java_thread() const { + return field_java_thread::decode(_flags); + } + + bool worker_thread() const { + return field_worker_thread::decode(_flags); + } + + bool non_blocking() const { + return field_non_blocking::decode(_flags); + } + + bool relocation() const { + return field_relocation::decode(_flags); + } + + bool no_reserve() const { + return field_no_reserve::decode(_flags); + } +}; + +#endif // SHARE_GC_Z_ZALLOCATIONFLAGS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zArguments.cpp 2018-06-01 22:30:19.455967799 +0200 @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/z/zArguments.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zCollectorPolicy.hpp" +#include "gc/z/zWorkers.hpp" +#include "gc/shared/gcArguments.inline.hpp" +#include "runtime/globals.hpp" +#include "runtime/globals_extension.hpp" + +size_t ZArguments::conservative_max_heap_alignment() { + return 0; +} + +void ZArguments::initialize() { + GCArguments::initialize(); + + // Enable NUMA by default + if (FLAG_IS_DEFAULT(UseNUMA)) { + FLAG_SET_DEFAULT(UseNUMA, true); + } + + // Disable biased locking by default + if (FLAG_IS_DEFAULT(UseBiasedLocking)) { + FLAG_SET_DEFAULT(UseBiasedLocking, false); + } + + // Select number of parallel threads + if (FLAG_IS_DEFAULT(ParallelGCThreads)) { + FLAG_SET_DEFAULT(ParallelGCThreads, ZWorkers::calculate_nparallel()); + } + + if (ParallelGCThreads == 0) { + vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ParallelGCThreads=0"); + } + + // Select number of concurrent threads + if (FLAG_IS_DEFAULT(ConcGCThreads)) { + FLAG_SET_DEFAULT(ConcGCThreads, ZWorkers::calculate_nconcurrent()); + } + + if (ConcGCThreads == 0) { + vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ConcGCThreads=0"); + } + +#ifdef COMPILER2 + // Enable loop strip mining by default + if (FLAG_IS_DEFAULT(UseCountedLoopSafepoints)) { + FLAG_SET_DEFAULT(UseCountedLoopSafepoints, true); + if (FLAG_IS_DEFAULT(LoopStripMiningIter)) { + FLAG_SET_DEFAULT(LoopStripMiningIter, 1000); + } + } +#endif + + // To avoid asserts in set_active_workers() + FLAG_SET_DEFAULT(UseDynamicNumberOfGCThreads, true); + + // CompressedOops/UseCompressedClassPointers not supported + FLAG_SET_DEFAULT(UseCompressedOops, false); + FLAG_SET_DEFAULT(UseCompressedClassPointers, false); + + // ClassUnloading not (yet) supported + FLAG_SET_DEFAULT(ClassUnloading, false); + FLAG_SET_DEFAULT(ClassUnloadingWithConcurrentMark, false); + + // Verification before startup and after exit not (yet) supported + FLAG_SET_DEFAULT(VerifyDuringStartup, false); + FLAG_SET_DEFAULT(VerifyBeforeExit, false); + + // JVMCI not (yet) supported + if (EnableJVMCI) { + vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:+EnableJVMCI"); + } +} + +CollectedHeap* ZArguments::create_heap() { + return create_heap_with_policy(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zArguments.hpp 2018-06-01 22:30:19.850984849 +0200 @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZARGUMENTS_HPP +#define SHARE_GC_Z_ZARGUMENTS_HPP + +#include "gc/shared/gcArguments.hpp" + +class CollectedHeap; + +class ZArguments : public GCArguments { +public: + virtual void initialize(); + virtual size_t conservative_max_heap_alignment(); + virtual CollectedHeap* create_heap(); +}; + +#endif // SHARE_GC_Z_ZARGUMENTS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zArray.hpp 2018-06-01 22:30:20.243001770 +0200 @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZARRAY_HPP +#define SHARE_GC_Z_ZARRAY_HPP + +#include "memory/allocation.hpp" + +template +class ZArray { +private: + static const size_t initial_capacity = 32; + + T* _array; + size_t _size; + size_t _capacity; + + // Copy and assignment are not allowed + ZArray(const ZArray& array); + ZArray& operator=(const ZArray& array); + + void expand(size_t new_capacity); + +public: + ZArray(); + ~ZArray(); + + size_t size() const; + bool is_empty() const; + + T at(size_t index) const; + + void add(T value); + void clear(); +}; + +template +class ZArrayIteratorImpl : public StackObj { +private: + ZArray* const _array; + size_t _next; + +public: + ZArrayIteratorImpl(ZArray* array); + + bool next(T* elem); +}; + +// Iterator types +#define ZARRAY_SERIAL false +#define ZARRAY_PARALLEL true + +template +class ZArrayIterator : public ZArrayIteratorImpl { +public: + ZArrayIterator(ZArray* array) : + ZArrayIteratorImpl(array) {} +}; + +template +class ZArrayParallelIterator : public ZArrayIteratorImpl { +public: + ZArrayParallelIterator(ZArray* array) : + ZArrayIteratorImpl(array) {} +}; + +#endif // SHARE_GC_Z_ZARRAY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zArray.inline.hpp 2018-06-01 22:30:20.630018476 +0200 @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZARRAY_INLINE_HPP +#define SHARE_GC_Z_ZARRAY_INLINE_HPP + +#include "gc/z/zArray.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/atomic.hpp" + +template +inline ZArray::ZArray() : + _array(NULL), + _size(0), + _capacity(0) {} + +template +inline ZArray::~ZArray() { + if (_array != NULL) { + FREE_C_HEAP_ARRAY(T, _array); + } +} + +template +inline size_t ZArray::size() const { + return _size; +} + +template +inline bool ZArray::is_empty() const { + return size() == 0; +} + +template +inline T ZArray::at(size_t index) const { + assert(index < _size, "Index out of bounds"); + return _array[index]; +} + +template +inline void ZArray::expand(size_t new_capacity) { + T* new_array = NEW_C_HEAP_ARRAY(T, new_capacity, mtGC); + if (_array != NULL) { + memcpy(new_array, _array, sizeof(T) * _capacity); + FREE_C_HEAP_ARRAY(T, _array); + } + + _array = new_array; + _capacity = new_capacity; +} + +template +inline void ZArray::add(T value) { + if (_size == _capacity) { + const size_t new_capacity = (_capacity > 0) ? _capacity * 2 : initial_capacity; + expand(new_capacity); + } + + _array[_size++] = value; +} + +template +inline void ZArray::clear() { + _size = 0; +} + +template +inline ZArrayIteratorImpl::ZArrayIteratorImpl(ZArray* array) : + _array(array), + _next(0) {} + +template +inline bool ZArrayIteratorImpl::next(T* elem) { + if (parallel) { + const size_t next = Atomic::add(1u, &_next) - 1u; + if (next < _array->size()) { + *elem = _array->at(next); + return true; + } + } else { + if (_next < _array->size()) { + *elem = _array->at(_next++); + return true; + } + } + + // No more elements + return false; +} + +#endif // SHARE_GC_Z_ZARRAY_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrier.cpp 2018-06-01 22:30:21.003034577 +0200 @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zOop.inline.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/safepoint.hpp" +#include "utilities/debug.hpp" + +bool ZBarrier::during_mark() { + return ZGlobalPhase == ZPhaseMark; +} + +bool ZBarrier::during_relocate() { + return ZGlobalPhase == ZPhaseRelocate; +} + +template +bool ZBarrier::should_mark_through(uintptr_t addr) { + // Finalizable marked oops can still exists on the heap after marking + // has completed, in which case we just want to convert this into a + // good oop and not push it on the mark stack. + if (!during_mark()) { + assert(ZAddress::is_marked(addr), "Should be marked"); + assert(ZAddress::is_finalizable(addr), "Should be finalizable"); + return false; + } + + // During marking, we mark through already marked oops to avoid having + // some large part of the object graph hidden behind a pushed, but not + // yet flushed, entry on a mutator mark stack. Always marking through + // allows the GC workers to proceed through the object graph even if a + // mutator touched an oop first, which in turn will reduce the risk of + // having to flush mark stacks multiple times to terminate marking. + // + // However, when doing finalizable marking we don't always want to mark + // through. First, marking through an already strongly marked oop would + // be wasteful, since we will then proceed to do finalizable marking on + // an object which is, or will be, marked strongly. Second, marking + // through an already finalizable marked oop would also be wasteful, + // since such oops can never end up on a mutator mark stack and can + // therefore not hide some part of the object graph from GC workers. + if (finalizable) { + return !ZAddress::is_marked(addr); + } + + // Mark through + return true; +} + +template +uintptr_t ZBarrier::mark(uintptr_t addr) { + uintptr_t good_addr; + + if (ZAddress::is_marked(addr)) { + // Already marked, but try to mark though anyway + good_addr = ZAddress::good(addr); + } else if (ZAddress::is_remapped(addr)) { + // Already remapped, but also needs to be marked + good_addr = ZAddress::good(addr); + } else { + // Needs to be both remapped and marked + good_addr = remap(addr); + } + + // Mark + if (should_mark_through(addr)) { + ZHeap::heap()->mark_object(good_addr); + } + + return good_addr; +} + +uintptr_t ZBarrier::remap(uintptr_t addr) { + assert(!ZAddress::is_good(addr), "Should not be good"); + assert(!ZAddress::is_weak_good(addr), "Should not be weak good"); + + if (ZHeap::heap()->is_relocating(addr)) { + // Forward + return ZHeap::heap()->forward_object(addr); + } + + // Remap + return ZAddress::good(addr); +} + +uintptr_t ZBarrier::relocate(uintptr_t addr) { + assert(!ZAddress::is_good(addr), "Should not be good"); + assert(!ZAddress::is_weak_good(addr), "Should not be weak good"); + + if (ZHeap::heap()->is_relocating(addr)) { + // Relocate + return ZHeap::heap()->relocate_object(addr); + } + + // Remap + return ZAddress::good(addr); +} + +uintptr_t ZBarrier::relocate_or_mark(uintptr_t addr) { + return during_relocate() ? relocate(addr) : mark(addr); +} + +uintptr_t ZBarrier::relocate_or_remap(uintptr_t addr) { + return during_relocate() ? relocate(addr) : remap(addr); +} + +// +// Load barrier +// +uintptr_t ZBarrier::load_barrier_on_oop_slow_path(uintptr_t addr) { + return relocate_or_mark(addr); +} + +void ZBarrier::load_barrier_on_oop_fields(oop o) { + assert(ZOop::is_good(o), "Should be good"); + ZLoadBarrierOopClosure cl; + o->oop_iterate(&cl); +} + +// +// Weak load barrier +// +uintptr_t ZBarrier::weak_load_barrier_on_oop_slow_path(uintptr_t addr) { + return ZAddress::is_weak_good(addr) ? ZAddress::good(addr) : relocate_or_remap(addr); +} + +uintptr_t ZBarrier::weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr) { + const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr); + if (ZHeap::heap()->is_object_strongly_live(good_addr)) { + return good_addr; + } + + // Not strongly live + return 0; +} + +uintptr_t ZBarrier::weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr) { + const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr); + if (ZHeap::heap()->is_object_live(good_addr)) { + return good_addr; + } + + // Not live + return 0; +} + +// +// Keep alive barrier +// +uintptr_t ZBarrier::keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr) { + const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr); + assert(ZHeap::heap()->is_object_strongly_live(good_addr), "Should be live"); + return good_addr; +} + +uintptr_t ZBarrier::keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr) { + const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr); + assert(ZHeap::heap()->is_object_live(good_addr), "Should be live"); + return good_addr; +} + +// +// Mark barrier +// +uintptr_t ZBarrier::mark_barrier_on_oop_slow_path(uintptr_t addr) { + return mark(addr); +} + +uintptr_t ZBarrier::mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr) { + const uintptr_t good_addr = mark(addr); + if (ZAddress::is_good(addr)) { + // If the oop was already strongly marked/good, then we do + // not want to downgrade it to finalizable marked/good. + return good_addr; + } + + // Make the oop finalizable marked/good, instead of normal marked/good. + // This is needed because an object might first becomes finalizable + // marked by the GC, and then loaded by a mutator thread. In this case, + // the mutator thread must be able to tell that the object needs to be + // strongly marked. The finalizable bit in the oop exists to make sure + // that a load of a finalizable marked oop will fall into the barrier + // slow path so that we can mark the object as strongly reachable. + return ZAddress::finalizable_good(good_addr); +} + +uintptr_t ZBarrier::mark_barrier_on_root_oop_slow_path(uintptr_t addr) { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + assert(during_mark(), "Invalid phase"); + + // Mark + return mark(addr); +} + +// +// Relocate barrier +// +uintptr_t ZBarrier::relocate_barrier_on_root_oop_slow_path(uintptr_t addr) { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + assert(during_relocate(), "Invalid phase"); + + // Relocate + return relocate(addr); +} + +// +// Narrow oop variants, never used. +// +oop ZBarrier::load_barrier_on_oop_field(volatile narrowOop* p) { + ShouldNotReachHere(); + return NULL; +} + +oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o) { + ShouldNotReachHere(); + return NULL; +} + +void ZBarrier::load_barrier_on_oop_array(volatile narrowOop* p, size_t length) { + ShouldNotReachHere(); +} + +oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o) { + ShouldNotReachHere(); + return NULL; +} + +oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o) { + ShouldNotReachHere(); + return NULL; +} + +oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o) { + ShouldNotReachHere(); + return NULL; +} + +oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o) { + ShouldNotReachHere(); + return NULL; +} + +oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o) { + ShouldNotReachHere(); + return NULL; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrier.hpp 2018-06-01 22:30:21.388051195 +0200 @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBARRIER_HPP +#define SHARE_GC_Z_ZBARRIER_HPP + +#include "memory/allocation.hpp" +#include "oops/oop.hpp" + +typedef bool (*ZBarrierFastPath)(uintptr_t); +typedef uintptr_t (*ZBarrierSlowPath)(uintptr_t); + +class ZBarrier : public AllStatic { +private: + static const bool Strong = false; + static const bool Finalizable = true; + + static const bool Publish = true; + static const bool Overflow = false; + + template static oop barrier(volatile oop* p, oop o); + template static oop weak_barrier(volatile oop* p, oop o); + template static void root_barrier(oop* p, oop o); + + static bool is_null_fast_path(uintptr_t addr); + static bool is_good_or_null_fast_path(uintptr_t addr); + static bool is_weak_good_or_null_fast_path(uintptr_t addr); + + static bool is_resurrection_blocked(volatile oop* p, oop* o); + + static bool during_mark(); + static bool during_relocate(); + template static bool should_mark_through(uintptr_t addr); + template static uintptr_t mark(uintptr_t addr); + static uintptr_t remap(uintptr_t addr); + static uintptr_t relocate(uintptr_t addr); + static uintptr_t relocate_or_mark(uintptr_t addr); + static uintptr_t relocate_or_remap(uintptr_t addr); + + static uintptr_t load_barrier_on_oop_slow_path(uintptr_t addr); + + static uintptr_t weak_load_barrier_on_oop_slow_path(uintptr_t addr); + static uintptr_t weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr); + static uintptr_t weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr); + + static uintptr_t keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr); + static uintptr_t keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr); + + static uintptr_t mark_barrier_on_oop_slow_path(uintptr_t addr); + static uintptr_t mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr); + static uintptr_t mark_barrier_on_root_oop_slow_path(uintptr_t addr); + + static uintptr_t relocate_barrier_on_root_oop_slow_path(uintptr_t addr); + +public: + // Load barrier + static oop load_barrier_on_oop(oop o); + static oop load_barrier_on_oop_field(volatile oop* p); + static oop load_barrier_on_oop_field_preloaded(volatile oop* p, oop o); + static void load_barrier_on_oop_array(volatile oop* p, size_t length); + static void load_barrier_on_oop_fields(oop o); + static oop load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o); + static oop load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o); + + // Weak load barrier + static oop weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o); + static oop weak_load_barrier_on_weak_oop(oop o); + static oop weak_load_barrier_on_weak_oop_field(volatile oop* p); + static oop weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o); + static oop weak_load_barrier_on_phantom_oop(oop o); + static oop weak_load_barrier_on_phantom_oop_field(volatile oop* p); + static oop weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o); + + // Is alive barrier + static bool is_alive_barrier_on_weak_oop(oop o); + static bool is_alive_barrier_on_phantom_oop(oop o); + + // Keep alive barrier + static void keep_alive_barrier_on_weak_oop_field(volatile oop* p); + static void keep_alive_barrier_on_phantom_oop_field(volatile oop* p); + + // Mark barrier + static void mark_barrier_on_oop_field(volatile oop* p, bool finalizable); + static void mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable); + static void mark_barrier_on_root_oop_field(oop* p); + + // Relocate barrier + static void relocate_barrier_on_root_oop_field(oop* p); + + // Narrow oop variants, never used. + static oop load_barrier_on_oop_field(volatile narrowOop* p); + static oop load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o); + static void load_barrier_on_oop_array(volatile narrowOop* p, size_t length); + static oop load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o); + static oop load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o); + static oop weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o); + static oop weak_load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o); + static oop weak_load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o); +}; + +#endif // SHARE_GC_Z_ZBARRIER_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrier.inline.hpp 2018-06-01 22:30:21.775067901 +0200 @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBARRIER_INLINE_HPP +#define SHARE_GC_Z_ZBARRIER_INLINE_HPP + +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zBarrier.hpp" +#include "gc/z/zOop.inline.hpp" +#include "gc/z/zResurrection.inline.hpp" +#include "runtime/atomic.hpp" + +template +inline oop ZBarrier::barrier(volatile oop* p, oop o) { + uintptr_t addr = ZOop::to_address(o); + +retry: + // Fast path + if (fast_path(addr)) { + return ZOop::to_oop(addr); + } + + // Slow path + const uintptr_t good_addr = slow_path(addr); + + // Self heal, but only if the address was actually updated by the slow path, + // which might not be the case, e.g. when marking through an already good oop. + if (p != NULL && good_addr != addr) { + const uintptr_t prev_addr = Atomic::cmpxchg(good_addr, (volatile uintptr_t*)p, addr); + if (prev_addr != addr) { + // Some other thread overwrote the oop. If this oop was updated by a + // weak barrier the new oop might not be good, in which case we need + // to re-apply this barrier. + addr = prev_addr; + goto retry; + } + } + + return ZOop::to_oop(good_addr); +} + +template +inline oop ZBarrier::weak_barrier(volatile oop* p, oop o) { + const uintptr_t addr = ZOop::to_address(o); + + // Fast path + if (fast_path(addr)) { + // Return the good address instead of the weak good address + // to ensure that the currently active heap view is used. + return ZOop::to_oop(ZAddress::good_or_null(addr)); + } + + // Slow path + uintptr_t good_addr = slow_path(addr); + + // Self heal unless the address returned from the slow path is null, + // in which case resurrection was blocked and we must let the reference + // processor clear the oop. Mutators are not allowed to clear oops in + // these cases, since that would be similar to calling Reference.clear(), + // which would make the reference non-discoverable or silently dropped + // by the reference processor. + if (p != NULL && good_addr != 0) { + // The slow path returns a good/marked address, but we never mark oops + // in a weak load barrier so we always self heal with the remapped address. + const uintptr_t weak_good_addr = ZAddress::remapped(good_addr); + const uintptr_t prev_addr = Atomic::cmpxchg(weak_good_addr, (volatile uintptr_t*)p, addr); + if (prev_addr != addr) { + // Some other thread overwrote the oop. The new + // oop is guaranteed to be weak good or null. + assert(ZAddress::is_weak_good_or_null(prev_addr), "Bad weak overwrite"); + + // Return the good address instead of the weak good address + // to ensure that the currently active heap view is used. + good_addr = ZAddress::good_or_null(prev_addr); + } + } + + return ZOop::to_oop(good_addr); +} + +template +inline void ZBarrier::root_barrier(oop* p, oop o) { + const uintptr_t addr = ZOop::to_address(o); + + // Fast path + if (fast_path(addr)) { + return; + } + + // Slow path + const uintptr_t good_addr = slow_path(addr); + + // Non-atomic healing helps speed up root scanning. This is safe to do + // since we are always healing roots in a safepoint, which means we are + // never racing with mutators modifying roots while we are healing them. + // It's also safe in case multiple GC threads try to heal the same root, + // since they would always heal the root in the same way and it does not + // matter in which order it happens. + *p = ZOop::to_oop(good_addr); +} + +inline bool ZBarrier::is_null_fast_path(uintptr_t addr) { + return ZAddress::is_null(addr); +} + +inline bool ZBarrier::is_good_or_null_fast_path(uintptr_t addr) { + return ZAddress::is_good_or_null(addr); +} + +inline bool ZBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) { + return ZAddress::is_weak_good_or_null(addr); +} + +inline bool ZBarrier::is_resurrection_blocked(volatile oop* p, oop* o) { + const bool is_blocked = ZResurrection::is_blocked(); + + // Reload oop after checking the resurrection blocked state. This is + // done to prevent a race where we first load an oop, which is logically + // null but not yet cleared, then this oop is cleared by the reference + // processor and resurrection is unblocked. At this point the mutator + // would see the unblocked state and pass this invalid oop through the + // normal barrier path, which would incorrectly try to mark this oop. + if (p != NULL) { + // First assign to reloaded_o to avoid compiler warning about + // implicit dereference of volatile oop. + const oop reloaded_o = *p; + *o = reloaded_o; + } + + return is_blocked; +} + +// +// Load barrier +// +inline oop ZBarrier::load_barrier_on_oop(oop o) { + return load_barrier_on_oop_field_preloaded((oop*)NULL, o); +} + +inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) { + const oop o = *p; + return load_barrier_on_oop_field_preloaded(p, o); +} + +inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) { + return barrier(p, o); +} + +inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) { + for (volatile const oop* const end = p + length; p < end; p++) { + load_barrier_on_oop_field(p); + } +} + +inline oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) { + if (is_resurrection_blocked(p, &o)) { + return weak_barrier(p, o); + } + + return load_barrier_on_oop_field_preloaded(p, o); +} + +inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) { + if (is_resurrection_blocked(p, &o)) { + return weak_barrier(p, o); + } + + return load_barrier_on_oop_field_preloaded(p, o); +} + +// +// Weak load barrier +// +inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) { + return weak_barrier(p, o); +} + +inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) { + return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o); +} + +inline oop ZBarrier::weak_load_barrier_on_weak_oop_field(volatile oop* p) { + const oop o = *p; + return weak_load_barrier_on_weak_oop_field_preloaded(p, o); +} + +inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) { + if (is_resurrection_blocked(p, &o)) { + return weak_barrier(p, o); + } + + return weak_load_barrier_on_oop_field_preloaded(p, o); +} + +inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) { + return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o); +} + +inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field(volatile oop* p) { + const oop o = *p; + return weak_load_barrier_on_phantom_oop_field_preloaded(p, o); +} + +inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) { + if (is_resurrection_blocked(p, &o)) { + return weak_barrier(p, o); + } + + return weak_load_barrier_on_oop_field_preloaded(p, o); +} + +// +// Is alive barrier +// +inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) { + // Check if oop is logically non-null. This operation + // is only valid when resurrection is blocked. + assert(ZResurrection::is_blocked(), "Invalid phase"); + return weak_load_barrier_on_weak_oop(o) != NULL; +} + +inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) { + // Check if oop is logically non-null. This operation + // is only valid when resurrection is blocked. + assert(ZResurrection::is_blocked(), "Invalid phase"); + return weak_load_barrier_on_phantom_oop(o) != NULL; +} + +// +// Keep alive barrier +// +inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) { + // This operation is only valid when resurrection is blocked. + assert(ZResurrection::is_blocked(), "Invalid phase"); + const oop o = *p; + barrier(p, o); +} + +inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) { + // This operation is only valid when resurrection is blocked. + assert(ZResurrection::is_blocked(), "Invalid phase"); + const oop o = *p; + barrier(p, o); +} + +// +// Mark barrier +// +inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) { + // The fast path only checks for null since the GC worker + // threads doing marking wants to mark through good oops. + const oop o = *p; + + if (finalizable) { + barrier(p, o); + } else { + barrier(p, o); + } +} + +inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) { + for (volatile const oop* const end = p + length; p < end; p++) { + mark_barrier_on_oop_field(p, finalizable); + } +} + +inline void ZBarrier::mark_barrier_on_root_oop_field(oop* p) { + const oop o = *p; + root_barrier(p, o); +} + +// +// Relocate barrier +// +inline void ZBarrier::relocate_barrier_on_root_oop_field(oop* p) { + const oop o = *p; + root_barrier(p, o); +} + +#endif // SHARE_GC_Z_ZBARRIER_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrierSet.cpp 2018-06-01 22:30:22.169084908 +0200 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/c1/zBarrierSetC1.hpp" +#include "gc/z/c2/zBarrierSetC2.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zThreadLocalData.hpp" +#include "runtime/thread.hpp" + +ZBarrierSet::ZBarrierSet() : + BarrierSet(make_barrier_set_assembler(), + make_barrier_set_c1(), + make_barrier_set_c2(), + BarrierSet::FakeRtti(BarrierSet::ZBarrierSet)) {} + +ZBarrierSetAssembler* ZBarrierSet::assembler() { + BarrierSetAssembler* const bsa = BarrierSet::barrier_set()->barrier_set_assembler(); + return reinterpret_cast(bsa); +} + +bool ZBarrierSet::barrier_needed(DecoratorSet decorators, BasicType type) { + assert((decorators & AS_RAW) == 0, "Unexpected decorator"); + assert((decorators & AS_NO_KEEPALIVE) == 0, "Unexpected decorator"); + assert((decorators & IN_ARCHIVE_ROOT) == 0, "Unexpected decorator"); + //assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Unexpected decorator"); + + if (type == T_OBJECT || type == T_ARRAY) { + if (((decorators & IN_HEAP) != 0) || + ((decorators & IN_CONCURRENT_ROOT) != 0) || + ((decorators & ON_PHANTOM_OOP_REF) != 0)) { + // Barrier needed + return true; + } + } + + // Barrier not neeed + return false; +} + +void ZBarrierSet::on_thread_create(Thread* thread) { + // Create thread local data + ZThreadLocalData::create(thread); +} + +void ZBarrierSet::on_thread_destroy(Thread* thread) { + // Destroy thread local data + ZThreadLocalData::destroy(thread); +} + +void ZBarrierSet::on_thread_attach(JavaThread* thread) { + // Set thread local address bad mask + ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask); +} + +void ZBarrierSet::on_thread_detach(JavaThread* thread) { + // Flush and free any remaining mark stacks + ZHeap::heap()->mark_flush_and_free(thread); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrierSet.hpp 2018-06-01 22:30:22.560101786 +0200 @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBARRIERSET_HPP +#define SHARE_GC_Z_ZBARRIERSET_HPP + +#include "gc/shared/barrierSet.hpp" + +class ZBarrierSetAssembler; + +class ZBarrierSet : public BarrierSet { +public: + ZBarrierSet(); + + static ZBarrierSetAssembler* assembler(); + static bool barrier_needed(DecoratorSet decorators, BasicType type); + + virtual void on_thread_create(Thread* thread); + virtual void on_thread_destroy(Thread* thread); + virtual void on_thread_attach(JavaThread* thread); + virtual void on_thread_detach(JavaThread* thread); + + virtual void print_on(outputStream* st) const {} + + template + class AccessBarrier : public BarrierSet::AccessBarrier { + private: + typedef BarrierSet::AccessBarrier Raw; + + template + static void verify_decorators_present(); + + template + static void verify_decorators_absent(); + + static oop* field_addr(oop base, ptrdiff_t offset); + + template + static oop load_barrier_on_oop_field_preloaded(T* addr, oop o); + + template + static oop load_barrier_on_unknown_oop_field_preloaded(oop base, ptrdiff_t offset, T* addr, oop o); + + public: + // + // In heap + // + template + static oop oop_load_in_heap(T* addr); + static oop oop_load_in_heap_at(oop base, ptrdiff_t offset); + + template + static oop oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value); + static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value); + + template + static oop oop_atomic_xchg_in_heap(oop new_value, T* addr); + static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset); + + template + static bool oop_arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length); + + static void clone_in_heap(oop src, oop dst, size_t size); + + // + // Not in heap + // + template + static oop oop_load_not_in_heap(T* addr); + + template + static oop oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value); + + template + static oop oop_atomic_xchg_not_in_heap(oop new_value, T* addr); + }; +}; + +template<> struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::ZBarrierSet; +}; + +template<> struct BarrierSet::GetType { + typedef ::ZBarrierSet type; +}; + +#endif // SHARE_GC_Z_ZBARRIERSET_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrierSet.inline.hpp 2018-06-01 22:30:22.946118448 +0200 @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP +#define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP + +#include "gc/shared/accessBarrierSupport.inline.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "utilities/debug.hpp" + +template +template +inline void ZBarrierSet::AccessBarrier::verify_decorators_present() { + if ((decorators & expected) == 0) { + fatal("Using unsupported access decorators"); + } +} + +template +template +inline void ZBarrierSet::AccessBarrier::verify_decorators_absent() { + if ((decorators & expected) != 0) { + fatal("Using unsupported access decorators"); + } +} + +template +inline oop* ZBarrierSet::AccessBarrier::field_addr(oop base, ptrdiff_t offset) { + assert(base != NULL, "Invalid base"); + return reinterpret_cast(reinterpret_cast((void*)base) + offset); +} + +template +template +inline oop ZBarrierSet::AccessBarrier::load_barrier_on_oop_field_preloaded(T* addr, oop o) { + verify_decorators_absent(); + + if (HasDecorator::value) { + if (HasDecorator::value) { + return ZBarrier::weak_load_barrier_on_oop_field_preloaded(addr, o); + } else if (HasDecorator::value) { + return ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(addr, o); + } else { + return ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(addr, o); + } + } else { + if (HasDecorator::value) { + return ZBarrier::load_barrier_on_oop_field_preloaded(addr, o); + } else if (HasDecorator::value) { + return ZBarrier::load_barrier_on_weak_oop_field_preloaded(addr, o); + } else { + return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(addr, o); + } + } +} + +template +template +inline oop ZBarrierSet::AccessBarrier::load_barrier_on_unknown_oop_field_preloaded(oop base, ptrdiff_t offset, T* addr, oop o) { + verify_decorators_present(); + + const DecoratorSet decorators_known_strength = + AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset); + + if (HasDecorator::value) { + if (decorators_known_strength & ON_STRONG_OOP_REF) { + return ZBarrier::weak_load_barrier_on_oop_field_preloaded(addr, o); + } else if (decorators_known_strength & ON_WEAK_OOP_REF) { + return ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(addr, o); + } else { + return ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(addr, o); + } + } else { + if (decorators_known_strength & ON_STRONG_OOP_REF) { + return ZBarrier::load_barrier_on_oop_field_preloaded(addr, o); + } else if (decorators_known_strength & ON_WEAK_OOP_REF) { + return ZBarrier::load_barrier_on_weak_oop_field_preloaded(addr, o); + } else { + return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(addr, o); + } + } +} + +// +// In heap +// +template +template +inline oop ZBarrierSet::AccessBarrier::oop_load_in_heap(T* addr) { + verify_decorators_absent(); + + const oop o = Raw::oop_load_in_heap(addr); + return load_barrier_on_oop_field_preloaded(addr, o); +} + +template +inline oop ZBarrierSet::AccessBarrier::oop_load_in_heap_at(oop base, ptrdiff_t offset) { + oop* const addr = field_addr(base, offset); + const oop o = Raw::oop_load_in_heap(addr); + + if (HasDecorator::value) { + return load_barrier_on_unknown_oop_field_preloaded(base, offset, addr, o); + } + + return load_barrier_on_oop_field_preloaded(addr, o); +} + +template +template +inline oop ZBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) { + verify_decorators_present(); + verify_decorators_absent(); + + ZBarrier::load_barrier_on_oop_field(addr); + return Raw::oop_atomic_cmpxchg_in_heap(new_value, addr, compare_value); +} + +template +inline oop ZBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { + verify_decorators_present(); + verify_decorators_absent(); + + // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can recieve + // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF, + // with the motivation that if you're doing Unsafe operations on a Reference.referent + // field, then you're on your own anyway. + ZBarrier::load_barrier_on_oop_field(field_addr(base, offset)); + return Raw::oop_atomic_cmpxchg_in_heap_at(new_value, base, offset, compare_value); +} + +template +template +inline oop ZBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(oop new_value, T* addr) { + verify_decorators_present(); + verify_decorators_absent(); + + const oop o = Raw::oop_atomic_xchg_in_heap(new_value, addr); + return ZBarrier::load_barrier_on_oop(o); +} + +template +inline oop ZBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { + verify_decorators_present(); + verify_decorators_absent(); + + const oop o = Raw::oop_atomic_xchg_in_heap_at(new_value, base, offset); + return ZBarrier::load_barrier_on_oop(o); +} + +template +template +inline bool ZBarrierSet::AccessBarrier::oop_arraycopy_in_heap(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + if (!HasDecorator::value) { + // No check cast, bulk barrier and bulk copy + ZBarrier::load_barrier_on_oop_array(src, length); + return Raw::oop_arraycopy_in_heap(src_obj, dst_obj, src, dst, length); + } + + // Check cast and copy each elements + Klass* const dst_klass = objArrayOop(dst_obj)->element_klass(); + for (const T* const end = src + length; src < end; src++, dst++) { + const oop elem = ZBarrier::load_barrier_on_oop_field(src); + if (!oopDesc::is_instanceof_or_null(elem, dst_klass)) { + // Check cast failed + return false; + } + + // Cast is safe, since we know it's never a narrowOop + *(oop*)dst = elem; + } + + return true; +} + +template +inline void ZBarrierSet::AccessBarrier::clone_in_heap(oop src, oop dst, size_t size) { + ZBarrier::load_barrier_on_oop_fields(src); + Raw::clone_in_heap(src, dst, size); +} + +// +// Not in heap +// +template +template +inline oop ZBarrierSet::AccessBarrier::oop_load_not_in_heap(T* addr) { + const oop o = Raw::oop_load_not_in_heap(addr); + + if (HasDecorator::value) { + return load_barrier_on_oop_field_preloaded(addr, o); + } + + verify_decorators_present(); + verify_decorators_absent(); + + return o; +} + +template +template +inline oop ZBarrierSet::AccessBarrier::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) { + verify_decorators_present(); + verify_decorators_absent(); + + return Raw::oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value); +} + +template +template +inline oop ZBarrierSet::AccessBarrier::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) { + verify_decorators_present(); + verify_decorators_absent(); + + return Raw::oop_atomic_xchg_not_in_heap(new_value, addr); +} + +#endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrierSetAssembler.cpp 2018-06-01 22:30:23.330135024 +0200 @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" +#include "gc/z/zThreadLocalData.hpp" +#include "runtime/thread.hpp" + +Address ZBarrierSetAssemblerBase::address_bad_mask_from_thread(Register thread) { + return Address(thread, ZThreadLocalData::address_bad_mask_offset()); +} + +Address ZBarrierSetAssemblerBase::address_bad_mask_from_jni_env(Register env) { + return Address(env, ZThreadLocalData::address_bad_mask_offset() - JavaThread::jni_environment_offset()); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp 2018-06-01 22:30:23.710151427 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP +#define SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "oops/accessDecorators.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +class ZBarrierSetAssemblerBase : public BarrierSetAssembler { +public: + static Address address_bad_mask_from_thread(Register thread); + static Address address_bad_mask_from_jni_env(Register env); +}; + +#include CPU_HEADER(gc/z/zBarrierSetAssembler) + +#endif // SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrierSetRuntime.cpp 2018-06-01 22:30:24.094168003 +0200 @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zBarrierSetRuntime.hpp" +#include "runtime/interfaceSupport.inline.hpp" + +JRT_LEAF(oopDesc*, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p)) + return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); +JRT_END + +JRT_LEAF(oopDesc*, ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p)) + return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o); +JRT_END + +JRT_LEAF(oopDesc*, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p)) + return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o); +JRT_END + +JRT_LEAF(void, ZBarrierSetRuntime::load_barrier_on_oop_array(oop* p, size_t length)) + ZBarrier::load_barrier_on_oop_array(p, length); +JRT_END + +address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators) { + if (decorators & ON_PHANTOM_OOP_REF) { + return load_barrier_on_phantom_oop_field_preloaded_addr(); + } else if (decorators & ON_WEAK_OOP_REF) { + return load_barrier_on_weak_oop_field_preloaded_addr(); + } else { + return load_barrier_on_oop_field_preloaded_addr(); + } +} + +address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() { + return reinterpret_cast
(load_barrier_on_oop_field_preloaded); +} + +address ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr() { + return reinterpret_cast
(load_barrier_on_weak_oop_field_preloaded); +} + +address ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr() { + return reinterpret_cast
(load_barrier_on_phantom_oop_field_preloaded); +} + +address ZBarrierSetRuntime::load_barrier_on_oop_array_addr() { + return reinterpret_cast
(load_barrier_on_oop_array); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp 2018-06-01 22:30:24.484184837 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP +#define SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP + +#include "memory/allocation.hpp" +#include "oops/accessDecorators.hpp" +#include "utilities/globalDefinitions.hpp" + +class oopDesc; + +class ZBarrierSetRuntime : public AllStatic { +private: + static oopDesc* load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p); + static oopDesc* load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p); + static oopDesc* load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p); + static void load_barrier_on_oop_array(oop* p, size_t length); + +public: + static address load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators); + static address load_barrier_on_oop_field_preloaded_addr(); + static address load_barrier_on_weak_oop_field_preloaded_addr(); + static address load_barrier_on_phantom_oop_field_preloaded_addr(); + static address load_barrier_on_oop_array_addr(); +}; + +#endif // SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBitField.hpp 2018-06-01 22:30:24.859201025 +0200 @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBITFIELD_HPP +#define SHARE_GC_Z_ZBITFIELD_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +// +// Example +// ------- +// +// typedef ZBitField field_word_aligned_size; +// typedef ZBitField field_length; +// +// +// 6 3 3 +// 3 2 1 2 10 +// +-----------------------------------+---------------------------------+--+ +// |11111111 11111111 11111111 11111111|11111111 11111111 11111111 111111|11| +// +-----------------------------------+---------------------------------+--+ +// | | | +// | 31-2 field_length (30-bits) * | +// | | +// | 1-0 field_word_aligned_size (2-bits) * +// | +// * 63-32 Unused (32-bits) +// +// +// field_word_aligned_size::encode(16) = 2 +// field_length::encode(2342) = 9368 +// +// field_word_aligned_size::decode(9368 | 2) = 16 +// field_length::decode(9368 | 2) = 2342 +// + +template +class ZBitField : public AllStatic { +private: + static const int ContainerBits = sizeof(ContainerType) * BitsPerByte; + + STATIC_ASSERT(FieldBits < ContainerBits); + STATIC_ASSERT(FieldShift + FieldBits <= ContainerBits); + STATIC_ASSERT(ValueShift + FieldBits <= ContainerBits); + + static const ContainerType FieldMask = (((ContainerType)1 << FieldBits) - 1); + +public: + static ValueType decode(ContainerType container) { + return (ValueType)(((container >> FieldShift) & FieldMask) << ValueShift); + } + + static ContainerType encode(ValueType value) { + assert(((ContainerType)value & (FieldMask << ValueShift)) == (ContainerType)value, "Invalid value"); + return ((ContainerType)value >> ValueShift) << FieldShift; + } +}; + +#endif // SHARE_GC_Z_ZBITFIELD_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBitMap.hpp 2018-06-01 22:30:25.237217341 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBITMAP_HPP +#define SHARE_GC_Z_ZBITMAP_HPP + +#include "utilities/bitMap.hpp" + +class ZBitMap : public CHeapBitMap { +private: + static bm_word_t bit_mask_pair(idx_t bit); + + bool par_set_bit_pair_finalizable(idx_t bit, bool& inc_live); + bool par_set_bit_pair_strong(idx_t bit, bool& inc_live); + +public: + ZBitMap(idx_t size_in_bits); + + bool par_set_bit_pair(idx_t bit, bool finalizable, bool& inc_live); +}; + +#endif // SHARE_GC_Z_ZBITMAP_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zBitMap.inline.hpp 2018-06-01 22:30:25.624234047 +0200 @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZBITMAP_INLINE_HPP +#define SHARE_GC_Z_ZBITMAP_INLINE_HPP + +#include "gc/z/zBitMap.hpp" +#include "runtime/atomic.hpp" +#include "utilities/bitMap.inline.hpp" +#include "utilities/debug.hpp" + +inline ZBitMap::ZBitMap(idx_t size_in_bits) : + CHeapBitMap(size_in_bits, mtGC, false /* clear */) {} + +inline BitMap::bm_word_t ZBitMap::bit_mask_pair(idx_t bit) { + assert(bit_in_word(bit) < BitsPerWord - 1, "Invalid bit index"); + return (bm_word_t)3 << bit_in_word(bit); +} + +inline bool ZBitMap::par_set_bit_pair_finalizable(idx_t bit, bool& inc_live) { + inc_live = par_set_bit(bit); + return inc_live; +} + +inline bool ZBitMap::par_set_bit_pair_strong(idx_t bit, bool& inc_live) { + verify_index(bit); + volatile bm_word_t* const addr = word_addr(bit); + const bm_word_t pair_mask = bit_mask_pair(bit); + bm_word_t old_val = *addr; + + do { + const bm_word_t new_val = old_val | pair_mask; + if (new_val == old_val) { + inc_live = false; + return false; // Someone else beat us to it. + } + const bm_word_t cur_val = Atomic::cmpxchg(new_val, addr, old_val); + if (cur_val == old_val) { + const bm_word_t marked_mask = bit_mask(bit); + inc_live = !(old_val & marked_mask); + return true; // Success. + } + old_val = cur_val; // The value changed, try again. + } while (true); +} + +inline bool ZBitMap::par_set_bit_pair(idx_t bit, bool finalizable, bool& inc_live) { + if (finalizable) { + return par_set_bit_pair_finalizable(bit, inc_live); + } else { + return par_set_bit_pair_strong(bit, inc_live); + } +} + +#endif // SHARE_GC_Z_ZBITMAP_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zCPU.cpp 2018-06-01 22:30:26.001250320 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zCPU.hpp" +#include "logging/log.hpp" +#include "memory/padded.inline.hpp" +#include "runtime/os.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/debug.hpp" + +#define ZCPU_UNKNOWN_AFFINITY (Thread*)-1; +#define ZCPU_UNKNOWN_SELF (Thread*)-2; + +PaddedEnd* ZCPU::_affinity = NULL; +__thread Thread* ZCPU::_self = ZCPU_UNKNOWN_SELF; +__thread uint32_t ZCPU::_cpu = 0; + +void ZCPU::initialize() { + assert(_affinity == NULL, "Already initialized"); + const uint32_t ncpus = count(); + + _affinity = PaddedArray::create_unfreeable(ncpus); + + for (uint32_t i = 0; i < ncpus; i++) { + _affinity[i]._thread = ZCPU_UNKNOWN_AFFINITY; + } + + log_info(gc, init)("CPUs: %u total, %u available", + os::processor_count(), + os::initial_active_processor_count()); +} + +uint32_t ZCPU::count() { + return os::processor_count(); +} + +uint32_t ZCPU::id() { + assert(_affinity != NULL, "Not initialized"); + + // Fast path + if (_affinity[_cpu]._thread == _self) { + return _cpu; + } + + // Slow path + _self = Thread::current(); + _cpu = os::processor_id(); + + // Update affinity table + _affinity[_cpu]._thread = _self; + + return _cpu; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zCPU.hpp 2018-06-01 22:30:26.385266896 +0200 @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZCPU_HPP +#define SHARE_GC_Z_ZCPU_HPP + +#include "memory/allocation.hpp" +#include "memory/padded.hpp" + +class Thread; + +class ZCPU : public AllStatic { +private: + struct ZCPUAffinity { + Thread* _thread; + }; + + static PaddedEnd* _affinity; + static __thread Thread* _self; + static __thread uint32_t _cpu; + +public: + static void initialize(); + + static uint32_t count(); + static uint32_t id(); +}; + +#endif // SHARE_GC_Z_ZCPU_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zCollectedHeap.cpp 2018-06-01 22:30:26.755282867 +0200 @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zServiceability.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "runtime/mutexLocker.hpp" + +ZCollectedHeap* ZCollectedHeap::heap() { + CollectedHeap* heap = Universe::heap(); + assert(heap != NULL, "Uninitialized access to ZCollectedHeap::heap()"); + assert(heap->kind() == CollectedHeap::Z, "Invalid name"); + return (ZCollectedHeap*)heap; +} + +ZCollectedHeap::ZCollectedHeap(ZCollectorPolicy* policy) : + _collector_policy(policy), + _soft_ref_policy(), + _barrier_set(), + _initialize(&_barrier_set), + _heap(), + _director(new ZDirector()), + _driver(new ZDriver()), + _stat(new ZStat()), + _runtime_workers() {} + +CollectedHeap::Name ZCollectedHeap::kind() const { + return CollectedHeap::Z; +} + +const char* ZCollectedHeap::name() const { + return ZGCName; +} + +jint ZCollectedHeap::initialize() { + if (!_heap.is_initialized()) { + return JNI_ENOMEM; + } + + initialize_reserved_region((HeapWord*)ZAddressReservedStart(), + (HeapWord*)ZAddressReservedEnd()); + + return JNI_OK; +} + +void ZCollectedHeap::initialize_serviceability() { + _heap.serviceability_initialize(); +} + +void ZCollectedHeap::stop() { + _director->stop(); + _driver->stop(); + _stat->stop(); +} + +CollectorPolicy* ZCollectedHeap::collector_policy() const { + return _collector_policy; +} + +SoftRefPolicy* ZCollectedHeap::soft_ref_policy() { + return &_soft_ref_policy; +} + +size_t ZCollectedHeap::max_capacity() const { + return _heap.max_capacity(); +} + +size_t ZCollectedHeap::capacity() const { + return _heap.capacity(); +} + +size_t ZCollectedHeap::used() const { + return _heap.used(); +} + +bool ZCollectedHeap::is_maximal_no_gc() const { + // Not supported + ShouldNotReachHere(); + return false; +} + +bool ZCollectedHeap::is_scavengable(oop obj) { + return false; +} + +bool ZCollectedHeap::is_in(const void* p) const { + return is_in_reserved(p) && _heap.is_in((uintptr_t)p); +} + +bool ZCollectedHeap::is_in_closed_subset(const void* p) const { + return is_in(p); +} + +HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { + const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(requested_size)); + const uintptr_t addr = _heap.alloc_tlab(size_in_bytes); + + if (addr != 0) { + *actual_size = requested_size; + } + + return (HeapWord*)addr; +} + +HeapWord* ZCollectedHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) { + const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(size)); + return (HeapWord*)_heap.alloc_object(size_in_bytes); +} + +MetaWord* ZCollectedHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, + size_t size, + Metaspace::MetadataType mdtype) { + MetaWord* result; + + // Start asynchronous GC + collect(GCCause::_metadata_GC_threshold); + + // Expand and retry allocation + result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype); + if (result != NULL) { + return result; + } + + // Start synchronous GC + collect(GCCause::_metadata_GC_clear_soft_refs); + + // Retry allocation + result = loader_data->metaspace_non_null()->allocate(size, mdtype); + if (result != NULL) { + return result; + } + + // Expand and retry allocation + result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype); + if (result != NULL) { + return result; + } + + // Out of memory + return NULL; +} + +void ZCollectedHeap::collect(GCCause::Cause cause) { + _driver->collect(cause); +} + +void ZCollectedHeap::collect_as_vm_thread(GCCause::Cause cause) { + // These collection requests are ignored since ZGC can't run a synchronous + // GC cycle from within the VM thread. This is considered benign, since the + // only GC causes comming in here should be heap dumper and heap inspector. + // However, neither the heap dumper nor the heap inspector really need a GC + // to happen, but the result of their heap iterations might in that case be + // less accurate since they might include objects that would otherwise have + // been collected by a GC. + assert(Thread::current()->is_VM_thread(), "Should be the VM thread"); + guarantee(cause == GCCause::_heap_dump || + cause == GCCause::_heap_inspection, "Invalid cause"); +} + +void ZCollectedHeap::do_full_collection(bool clear_all_soft_refs) { + // Not supported + ShouldNotReachHere(); +} + +bool ZCollectedHeap::supports_tlab_allocation() const { + return true; +} + +size_t ZCollectedHeap::tlab_capacity(Thread* ignored) const { + return _heap.tlab_capacity(); +} + +size_t ZCollectedHeap::tlab_used(Thread* ignored) const { + return _heap.tlab_used(); +} + +size_t ZCollectedHeap::max_tlab_size() const { + return _heap.max_tlab_size(); +} + +size_t ZCollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const { + return _heap.unsafe_max_tlab_alloc(); +} + +bool ZCollectedHeap::can_elide_tlab_store_barriers() const { + return false; +} + +bool ZCollectedHeap::can_elide_initializing_store_barrier(oop new_obj) { + // Not supported + ShouldNotReachHere(); + return true; +} + +bool ZCollectedHeap::card_mark_must_follow_store() const { + // Not supported + ShouldNotReachHere(); + return false; +} + +GrowableArray ZCollectedHeap::memory_managers() { + return GrowableArray(1, 1, _heap.serviceability_memory_manager()); +} + +GrowableArray ZCollectedHeap::memory_pools() { + return GrowableArray(1, 1, _heap.serviceability_memory_pool()); +} + +void ZCollectedHeap::object_iterate(ObjectClosure* cl) { + _heap.object_iterate(cl); +} + +void ZCollectedHeap::safe_object_iterate(ObjectClosure* cl) { + _heap.object_iterate(cl); +} + +HeapWord* ZCollectedHeap::block_start(const void* addr) const { + return (HeapWord*)_heap.block_start((uintptr_t)addr); +} + +size_t ZCollectedHeap::block_size(const HeapWord* addr) const { + size_t size_in_bytes = _heap.block_size((uintptr_t)addr); + return ZUtils::bytes_to_words(size_in_bytes); +} + +bool ZCollectedHeap::block_is_obj(const HeapWord* addr) const { + return _heap.block_is_obj((uintptr_t)addr); +} + +void ZCollectedHeap::register_nmethod(nmethod* nm) { + assert_locked_or_safepoint(CodeCache_lock); + ZNMethodTable::register_nmethod(nm); +} + +void ZCollectedHeap::unregister_nmethod(nmethod* nm) { + assert_locked_or_safepoint(CodeCache_lock); + ZNMethodTable::unregister_nmethod(nm); +} + +void ZCollectedHeap::verify_nmethod(nmethod* nm) { + // Does nothing +} + +WorkGang* ZCollectedHeap::get_safepoint_workers() { + return _runtime_workers.workers(); +} + +jlong ZCollectedHeap::millis_since_last_gc() { + return ZStatCycle::time_since_last() / MILLIUNITS; +} + +void ZCollectedHeap::gc_threads_do(ThreadClosure* tc) const { + tc->do_thread(_director); + tc->do_thread(_driver); + tc->do_thread(_stat); + _heap.worker_threads_do(tc); + _runtime_workers.threads_do(tc); +} + +VirtualSpaceSummary ZCollectedHeap::create_heap_space_summary() { + const size_t capacity_in_words = capacity() / HeapWordSize; + const size_t max_capacity_in_words = max_capacity() / HeapWordSize; + return VirtualSpaceSummary(reserved_region().start(), + reserved_region().start() + capacity_in_words, + reserved_region().start() + max_capacity_in_words); +} + +void ZCollectedHeap::prepare_for_verify() { + // Does nothing +} + +void ZCollectedHeap::print_on(outputStream* st) const { + _heap.print_on(st); +} + +void ZCollectedHeap::print_on_error(outputStream* st) const { + CollectedHeap::print_on_error(st); + + st->print_cr("Address Space"); + st->print_cr( " Start: " PTR_FORMAT, ZAddressSpaceStart); + st->print_cr( " End: " PTR_FORMAT, ZAddressSpaceEnd); + st->print_cr( " Size: " SIZE_FORMAT_W(-15) " (" PTR_FORMAT ")", ZAddressSpaceSize, ZAddressSpaceSize); + st->print_cr( "Heap"); + st->print_cr( " GlobalPhase: %u", ZGlobalPhase); + st->print_cr( " GlobalSeqNum: %u", ZGlobalSeqNum); + st->print_cr( " Offset Max: " SIZE_FORMAT_W(-15) " (" PTR_FORMAT ")", ZAddressOffsetMax, ZAddressOffsetMax); + st->print_cr( " Page Size Small: " SIZE_FORMAT_W(-15) " (" PTR_FORMAT ")", ZPageSizeSmall, ZPageSizeSmall); + st->print_cr( " Page Size Medium: " SIZE_FORMAT_W(-15) " (" PTR_FORMAT ")", ZPageSizeMedium, ZPageSizeMedium); + st->print_cr( "Metadata Bits"); + st->print_cr( " Good: " PTR_FORMAT, ZAddressGoodMask); + st->print_cr( " Bad: " PTR_FORMAT, ZAddressBadMask); + st->print_cr( " WeakBad: " PTR_FORMAT, ZAddressWeakBadMask); + st->print_cr( " Marked: " PTR_FORMAT, ZAddressMetadataMarked); + st->print_cr( " Remapped: " PTR_FORMAT, ZAddressMetadataRemapped); +} + +void ZCollectedHeap::print_extended_on(outputStream* st) const { + _heap.print_extended_on(st); +} + +void ZCollectedHeap::print_gc_threads_on(outputStream* st) const { + _director->print_on(st); + st->cr(); + _driver->print_on(st); + st->cr(); + _stat->print_on(st); + st->cr(); + _heap.print_worker_threads_on(st); + _runtime_workers.print_threads_on(st); +} + +void ZCollectedHeap::print_tracing_info() const { + // Does nothing +} + +void ZCollectedHeap::verify(VerifyOption option /* ignored */) { + _heap.verify(); +} + +bool ZCollectedHeap::is_oop(oop object) const { + return CollectedHeap::is_oop(object) && _heap.is_oop(object); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zCollectedHeap.hpp 2018-06-01 22:30:27.130299055 +0200 @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZCOLLECTEDHEAP_HPP +#define SHARE_GC_Z_ZCOLLECTEDHEAP_HPP + +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/softRefPolicy.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zCollectorPolicy.hpp" +#include "gc/z/zDirector.hpp" +#include "gc/z/zDriver.hpp" +#include "gc/z/zInitialize.hpp" +#include "gc/z/zHeap.hpp" +#include "gc/z/zRuntimeWorkers.hpp" +#include "gc/z/zStat.hpp" + +class ZCollectedHeap : public CollectedHeap { + friend class VMStructs; + +private: + ZCollectorPolicy* _collector_policy; + SoftRefPolicy _soft_ref_policy; + ZBarrierSet _barrier_set; + ZInitialize _initialize; + ZHeap _heap; + ZDirector* _director; + ZDriver* _driver; + ZStat* _stat; + ZRuntimeWorkers _runtime_workers; + + virtual HeapWord* allocate_new_tlab(size_t min_size, + size_t requested_size, + size_t* actual_size); + +public: + static ZCollectedHeap* heap(); + + using CollectedHeap::ensure_parsability; + using CollectedHeap::accumulate_statistics_all_tlabs; + using CollectedHeap::resize_all_tlabs; + + ZCollectedHeap(ZCollectorPolicy* policy); + virtual Name kind() const; + virtual const char* name() const; + virtual jint initialize(); + virtual void initialize_serviceability(); + virtual void stop(); + + virtual CollectorPolicy* collector_policy() const; + virtual SoftRefPolicy* soft_ref_policy(); + + virtual size_t max_capacity() const; + virtual size_t capacity() const; + virtual size_t used() const; + + virtual bool is_maximal_no_gc() const; + virtual bool is_scavengable(oop obj); + virtual bool is_in(const void* p) const; + virtual bool is_in_closed_subset(const void* p) const; + + virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); + virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, + size_t size, + Metaspace::MetadataType mdtype); + virtual void collect(GCCause::Cause cause); + virtual void collect_as_vm_thread(GCCause::Cause cause); + virtual void do_full_collection(bool clear_all_soft_refs); + + virtual bool supports_tlab_allocation() const; + virtual size_t tlab_capacity(Thread* thr) const; + virtual size_t tlab_used(Thread* thr) const; + virtual size_t max_tlab_size() const; + virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; + + virtual bool can_elide_tlab_store_barriers() const; + virtual bool can_elide_initializing_store_barrier(oop new_obj); + virtual bool card_mark_must_follow_store() const; + + virtual GrowableArray memory_managers(); + virtual GrowableArray memory_pools(); + + virtual void object_iterate(ObjectClosure* cl); + virtual void safe_object_iterate(ObjectClosure* cl); + + virtual HeapWord* block_start(const void* addr) const; + virtual size_t block_size(const HeapWord* addr) const; + virtual bool block_is_obj(const HeapWord* addr) const; + + virtual void register_nmethod(nmethod* nm); + virtual void unregister_nmethod(nmethod* nm); + virtual void verify_nmethod(nmethod* nmethod); + + virtual WorkGang* get_safepoint_workers(); + + virtual jlong millis_since_last_gc(); + + virtual void gc_threads_do(ThreadClosure* tc) const; + + virtual VirtualSpaceSummary create_heap_space_summary(); + + virtual void print_on(outputStream* st) const; + virtual void print_on_error(outputStream* st) const; + virtual void print_extended_on(outputStream* st) const; + virtual void print_gc_threads_on(outputStream* st) const; + virtual void print_tracing_info() const; + + virtual void prepare_for_verify(); + virtual void verify(VerifyOption option /* ignored */); + virtual bool is_oop(oop object) const; +}; + +#endif // SHARE_GC_Z_ZCOLLECTEDHEAP_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zCollectorPolicy.cpp 2018-06-01 22:30:27.513315587 +0200 @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zCollectorPolicy.hpp" +#include "gc/z/zGlobals.hpp" + +void ZCollectorPolicy::initialize_alignments() { + _space_alignment = ZPageSizeMin; + _heap_alignment = _space_alignment; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zCollectorPolicy.hpp 2018-06-01 22:30:27.885331645 +0200 @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZCOLLECTORPOLICY_HPP +#define SHARE_GC_Z_ZCOLLECTORPOLICY_HPP + +#include "gc/shared/collectorPolicy.hpp" + +class ZCollectorPolicy : public CollectorPolicy { +public: + virtual void initialize_alignments(); +}; + +#endif // SHARE_GC_Z_ZCOLLECTORPOLICY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zDebug.gdb 2018-06-01 22:30:28.268348177 +0200 @@ -0,0 +1,147 @@ +# +# GDB functions for debugging the Z Garbage Collector +# + +printf "Loading zDebug.gdb\n" + +# Print Klass* +define zpk + printf "Klass: %s\n", (char*)((Klass*)($arg0))->_name->_body +end + +# Print oop +define zpo + set $obj = (oopDesc*)($arg0) + + printf "Oop: 0x%016llx\tState: ", (uintptr_t)$obj + if ((uintptr_t)$obj & (uintptr_t)ZAddressGoodMask) + printf "Good " + if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataRemapped) + printf "(Remapped)" + else + if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataMarked) + printf "(Marked)" + else + printf "(Unknown)" + end + end + else + printf "Bad " + if ((uintptr_t)ZAddressGoodMask & (uintptr_t)ZAddressMetadataMarked) + # Should be marked + if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataRemapped) + printf "(Not Marked, Remapped)" + else + printf "(Not Marked, Not Remapped)" + end + else + if ((uintptr_t)ZAddressGoodMask & (uintptr_t)ZAddressMetadataRemapped) + # Should be remapped + if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataMarked) + printf "(Marked, Not Remapped)" + else + printf "(Not Marked, Not Remapped)" + end + else + # Unknown + printf "(Unknown)" + end + end + end + printf "\t Page: %llu\n", ((uintptr_t)$obj & ZAddressOffsetMask) >> ZPageSizeMinShift + x/16gx $obj + printf "Mark: 0x%016llx\tKlass: %s\n", (uintptr_t)$obj->_mark, (char*)$obj->_metadata->_klass->_name->_body +end + +# Print heap page by pagetable index +define zpp + set $page = (ZPage*)((uintptr_t)ZHeap::_heap._pagetable._map._map[($arg0)] & ~1) + printf "Page %p\n", $page + print *$page +end + +# Print pagetable +define zpt + printf "Pagetable (first 128 slots)\n" + x/128gx ZHeap::_heap._pagetable._map._map +end + +# Print live map +define __zmarked + set $livemap = $arg0 + set $bit = $arg1 + set $size = $livemap._bitmap._size + set $segment = $size / ZLiveMap::nsegments + set $segment_bit = 1 << $segment + + printf "Segment is " + if !($livemap._segment_live_bits & $segment_bit) + printf "NOT " + end + printf "live (segment %d)\n", $segment + + if $bit >= $size + print "Error: Bit %z out of bounds (bitmap size %z)\n", $bit, $size + else + set $word_index = $bit / 64 + set $bit_index = $bit % 64 + set $word = $livemap._bitmap._map[$word_index] + set $live_bit = $word & (1 << $bit_index) + + printf "Object is " + if $live_bit == 0 + printf "NOT " + end + printf "live (word index %d, bit index %d)\n", $word_index, $bit_index + end +end + +define zmarked + set $addr = $arg0 + set $obj = ((uintptr_t)$addr & ZAddressOffsetMask) + set $page_index = $obj >> ZPageSizeMinShift + set $page_entry = (uintptr_t)ZHeap::_heap._pagetable._map._map[$page_index] + set $page = (ZPage*)($page_entry & ~1) + set $page_start = (uintptr_t)$page._virtual._start + set $page_end = (uintptr_t)$page._virtual._end + set $page_seqnum = $page._livemap._seqnum + set $global_seqnum = ZGlobalSeqNum + + if $obj < $page_start || $obj >= $page_end + printf "Error: %p not in page %p (start %p, end %p)\n", $obj, $page, $page_start, $page_end + else + printf "Page is " + if $page_seqnum != $global_seqnum + printf "NOT " + end + printf "live (page %p, page seqnum %d, global seqnum %d)\n", $page, $page_seqnum, $global_seqnum + + #if $page_seqnum == $global_seqnum + set $offset = $obj - $page_start + set $bit = $offset / 8 + __zmarked $page._livemap $bit + #end + end +end + +# Print heap information +define zph + printf "Address Space\n" + printf " Start: 0x%llx\n", ZAddressSpaceStart + printf " End: 0x%llx\n", ZAddressSpaceEnd + printf " Size: %-15llu (0x%llx)\n", ZAddressSpaceSize, ZAddressSpaceSize + printf "Heap\n" + printf " GlobalPhase: %u\n", ZGlobalPhase + printf " GlobalSeqNum: %u\n", ZGlobalSeqNum + printf " Offset Max: %-15llu (0x%llx)\n", ZAddressOffsetMax, ZAddressOffsetMax + printf " Page Size Small: %-15llu (0x%llx)\n", ZPageSizeSmall, ZPageSizeSmall + printf " Page Size Medium: %-15llu (0x%llx)\n", ZPageSizeMedium, ZPageSizeMedium + printf "Metadata Bits\n" + printf " Good: 0x%016llx\n", ZAddressGoodMask + printf " Bad: 0x%016llx\n", ZAddressBadMask + printf " WeakBad: 0x%016llx\n", ZAddressWeakBadMask + printf " Marked: 0x%016llx\n", ZAddressMetadataMarked + printf " Remapped: 0x%016llx\n", ZAddressMetadataRemapped +end + +# End of file --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zDirector.cpp 2018-06-01 22:30:28.660365099 +0200 @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zDirector.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zUtils.hpp" +#include "logging/log.hpp" + +const double ZDirector::one_in_1000 = 3.290527; + +ZDirector::ZDirector() : + _metronome(ZStatAllocRate::sample_hz) { + set_name("ZDirector"); + create_and_start(); +} + +void ZDirector::sample_allocation_rate() const { + // Sample allocation rate. This is needed by rule_allocation_rate() + // below to estimate the time we have until we run out of memory. + const double bytes_per_second = ZStatAllocRate::sample_and_reset(); + + log_debug(gc, alloc)("Allocation Rate: %.3fMB/s, Avg: %.3f(+/-%.3f)MB/s", + bytes_per_second / M, + ZStatAllocRate::avg() / M, + ZStatAllocRate::avg_sd() / M); +} + +bool ZDirector::is_first() const { + return ZStatCycle::ncycles() == 0; +} + +bool ZDirector::is_warm() const { + return ZStatCycle::ncycles() >= 3; +} + +bool ZDirector::rule_timer() const { + if (ZCollectionInterval == 0) { + // Rule disabled + return false; + } + + // Perform GC if timer has expired. + const double time_since_last_gc = ZStatCycle::time_since_last(); + const double time_until_gc = ZCollectionInterval - time_since_last_gc; + + log_debug(gc, director)("Rule: Timer, Interval: %us, TimeUntilGC: %.3lfs", + ZCollectionInterval, time_until_gc); + + return time_until_gc <= 0; +} + +bool ZDirector::rule_warmup() const { + if (is_warm()) { + // Rule disabled + return false; + } + + // Perform GC if heap usage passes 10/20/30% and no other GC has been + // performed yet. This allows us to get some early samples of the GC + // duration, which is needed by the other rules. + const size_t max_capacity = ZHeap::heap()->max_capacity(); + const size_t used = ZHeap::heap()->used(); + const double used_threshold_percent = (ZStatCycle::ncycles() + 1) * 0.1; + const size_t used_threshold = max_capacity * used_threshold_percent; + + log_debug(gc, director)("Rule: Warmup %.0f%%, Used: " SIZE_FORMAT "MB, UsedThreshold: " SIZE_FORMAT "MB", + used_threshold_percent * 100, used / M, used_threshold / M); + + return used >= used_threshold; +} + +bool ZDirector::rule_allocation_rate() const { + if (is_first()) { + // Rule disabled + return false; + } + + // Perform GC if the estimated max allocation rate indicates that we + // will run out of memory. The estimated max allocation rate is based + // on the moving average of the sampled allocation rate plus a safety + // margin based on variations in the allocation rate and unforseen + // allocation spikes. + + // Calculate amount of free memory available to Java threads. Note that + // the heap reserve is not available to Java threads and is therefore not + // considered part of the free memory. + const size_t max_capacity = ZHeap::heap()->max_capacity(); + const size_t max_reserve = ZHeap::heap()->max_reserve(); + const size_t used = ZHeap::heap()->used(); + const size_t free_with_reserve = max_capacity - used; + const size_t free = free_with_reserve - MIN2(free_with_reserve, max_reserve); + + // Calculate time until OOM given the max allocation rate and the amount + // of free memory. The allocation rate is a moving average and we multiply + // that with an alllcation spike tolerance factor to guard against unforseen + // phase changes in the allocate rate. We then add ~3.3 sigma to account for + // the allocation rate variance, which means the probablility is 1 in 1000 + // that a sample is outside of the confidence interval. + const double max_alloc_rate = (ZStatAllocRate::avg() * ZAllocationSpikeTolerance) + (ZStatAllocRate::avg_sd() * one_in_1000); + const double time_until_oom = free / (max_alloc_rate + 1.0); // Plus 1.0B/s to avoid division by zero + + // Calculate max duration of a GC cycle. The duration of GC is a moving + // average, we add ~3.3 sigma to account for the GC duration variance. + const AbsSeq& duration_of_gc = ZStatCycle::normalized_duration(); + const double max_duration_of_gc = duration_of_gc.davg() + (duration_of_gc.dsd() * one_in_1000); + + // Calculate time until GC given the time until OOM and max duration of GC. + // We also deduct the sample interval, so that we don't overshoot the target + // time and end up starting the GC too late in the next interval. + const double sample_interval = 1.0 / ZStatAllocRate::sample_hz; + const double time_until_gc = time_until_oom - max_duration_of_gc - sample_interval; + + log_debug(gc, director)("Rule: Allocation Rate, MaxAllocRate: %.3lfMB/s, Free: " SIZE_FORMAT "MB, MaxDurationOfGC: %.3lfs, TimeUntilGC: %.3lfs", + max_alloc_rate / M, free / M, max_duration_of_gc, time_until_gc); + + return time_until_gc <= 0; +} + +bool ZDirector::rule_proactive() const { + if (!is_warm()) { + // Rule disabled + return false; + } + + // Perform GC if the impact of doing so, in terms of application throughput + // reduction, is considered acceptable. This rule allows us to keep the heap + // size down and allow reference processing to happen even when we have a lot + // of free space on the heap. + + // Only consider doing a proactive GC if the heap usage has grown by at least + // 10% of the max capacity since the previous GC, or more than 5 minutes has + // passed since the previous GC. This helps avoid superfluous GCs when running + // applications with very low allocation rate. + const size_t used_after_last_gc = ZStatHeap::used_at_relocate_end(); + const size_t used_increase_threshold = ZHeap::heap()->max_capacity() * 0.10; // 10% + const size_t used_threshold = used_after_last_gc + used_increase_threshold; + const size_t used = ZHeap::heap()->used(); + const double time_since_last_gc = ZStatCycle::time_since_last(); + const double time_since_last_gc_threshold = 5 * 60; // 5 minutes + if (used < used_threshold && time_since_last_gc < time_since_last_gc_threshold) { + // Don't even consider doing a proactive GC + log_debug(gc, director)("Rule: Proactive, UsedUntilEnabled: " SIZE_FORMAT "MB, TimeUntilEnabled: %.3lfs", + (used_threshold - used) / M, + time_since_last_gc_threshold - time_since_last_gc); + return false; + } + + const double assumed_throughput_drop_during_gc = 0.50; // 50% + const double acceptable_throughput_drop = 0.01; // 1% + const AbsSeq& duration_of_gc = ZStatCycle::normalized_duration(); + const double max_duration_of_gc = duration_of_gc.davg() + (duration_of_gc.dsd() * one_in_1000); + const double acceptable_gc_interval = max_duration_of_gc * ((assumed_throughput_drop_during_gc / acceptable_throughput_drop) - 1.0); + const double time_until_gc = acceptable_gc_interval - time_since_last_gc; + + log_debug(gc, director)("Rule: Proactive, AcceptableGCInterval: %.3lfs, TimeSinceLastGC: %.3lfs, TimeUntilGC: %.3lfs", + acceptable_gc_interval, time_since_last_gc, time_until_gc); + + return time_until_gc <= 0; +} + +GCCause::Cause ZDirector::make_gc_decision() const { + // Rule 0: Timer + if (rule_timer()) { + return GCCause::_z_timer; + } + + // Rule 1: Warmup + if (rule_warmup()) { + return GCCause::_z_warmup; + } + + // Rule 2: Allocation rate + if (rule_allocation_rate()) { + return GCCause::_z_allocation_rate; + } + + // Rule 3: Proactive + if (rule_proactive()) { + return GCCause::_z_proactive; + } + + // No GC + return GCCause::_no_gc; +} + +void ZDirector::run_service() { + // Main loop + while (_metronome.wait_for_tick()) { + sample_allocation_rate(); + const GCCause::Cause cause = make_gc_decision(); + if (cause != GCCause::_no_gc) { + ZCollectedHeap::heap()->collect(cause); + } + } +} + +void ZDirector::stop_service() { + _metronome.stop(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zDirector.hpp 2018-06-01 22:30:29.044381674 +0200 @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZDIRECTOR_HPP +#define SHARE_GC_Z_ZDIRECTOR_HPP + +#include "gc/shared/concurrentGCThread.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/z/zMetronome.hpp" + +class ZDirector : public ConcurrentGCThread { +private: + static const double one_in_1000; + + ZMetronome _metronome; + + void sample_allocation_rate() const; + + bool is_first() const; + bool is_warm() const; + + bool rule_timer() const; + bool rule_warmup() const; + bool rule_allocation_rate() const; + bool rule_proactive() const; + GCCause::Cause make_gc_decision() const; + +protected: + virtual void run_service(); + virtual void stop_service(); + +public: + ZDirector(); +}; + +#endif // SHARE_GC_Z_ZDIRECTOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zDriver.cpp 2018-06-01 22:30:29.421397948 +0200 @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/gcId.hpp" +#include "gc/shared/gcLocker.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zDriver.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zMessagePort.inline.hpp" +#include "gc/z/zServiceability.hpp" +#include "gc/z/zStat.hpp" +#include "logging/log.hpp" +#include "runtime/vm_operations.hpp" +#include "runtime/vmThread.hpp" + +static const ZStatPhaseCycle ZPhaseCycle("Garbage Collection Cycle"); +static const ZStatPhasePause ZPhasePauseMarkStart("Pause Mark Start"); +static const ZStatPhaseConcurrent ZPhaseConcurrentMark("Concurrent Mark"); +static const ZStatPhaseConcurrent ZPhaseConcurrentMarkContinue("Concurrent Mark Continue"); +static const ZStatPhasePause ZPhasePauseMarkEnd("Pause Mark End"); +static const ZStatPhaseConcurrent ZPhaseConcurrentProcessNonStrongReferences("Concurrent Process Non-Strong References"); +static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSet("Concurrent Reset Relocation Set"); +static const ZStatPhaseConcurrent ZPhaseConcurrentDestroyDetachedPages("Concurrent Destroy Detached Pages"); +static const ZStatPhaseConcurrent ZPhaseConcurrentSelectRelocationSet("Concurrent Select Relocation Set"); +static const ZStatPhaseConcurrent ZPhaseConcurrentPrepareRelocationSet("Concurrent Prepare Relocation Set"); +static const ZStatPhasePause ZPhasePauseRelocateStart("Pause Relocate Start"); +static const ZStatPhaseConcurrent ZPhaseConcurrentRelocated("Concurrent Relocate"); +static const ZStatCriticalPhase ZCriticalPhaseGCLockerStall("GC Locker Stall", false /* verbose */); +static const ZStatSampler ZSamplerJavaThreads("System", "Java Threads", ZStatUnitThreads); + +class ZOperationClosure : public StackObj { +public: + virtual const char* name() const = 0; + + virtual bool needs_inactive_gc_locker() const { + // An inactive GC locker is needed in operations where we change the good + // mask or move objects. Changing the good mask will invalidate all oops, + // which makes it conceptually the same thing as moving all objects. + return false; + } + + virtual bool do_operation() = 0; +}; + +class VM_ZOperation : public VM_Operation { +private: + ZOperationClosure* _cl; + uint _gc_id; + bool _gc_locked; + bool _success; + +public: + VM_ZOperation(ZOperationClosure* cl) : + _cl(cl), + _gc_id(GCId::current()), + _gc_locked(false), + _success(false) {} + + virtual VMOp_Type type() const { + return VMOp_ZOperation; + } + + virtual const char* name() const { + return _cl->name(); + } + + virtual bool doit_prologue() { + Heap_lock->lock(); + return true; + } + + virtual void doit() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + + ZStatSample(ZSamplerJavaThreads, Threads::number_of_threads()); + + // JVMTI support + SvcGCMarker sgcm(SvcGCMarker::OTHER); + + // Setup GC id + GCIdMark gcid(_gc_id); + + if (_cl->needs_inactive_gc_locker() && GCLocker::check_active_before_gc()) { + // GC locker is active, bail out + _gc_locked = true; + } else { + // Execute operation + IsGCActiveMark mark; + _success = _cl->do_operation(); + } + } + + virtual void doit_epilogue() { + Heap_lock->unlock(); + } + + bool gc_locked() { + return _gc_locked; + } + + bool success() const { + return _success; + } +}; + +class ZMarkStartClosure : public ZOperationClosure { +public: + virtual const char* name() const { + return "ZMarkStart"; + } + + virtual bool needs_inactive_gc_locker() const { + return true; + } + + virtual bool do_operation() { + ZStatTimer timer(ZPhasePauseMarkStart); + ZServiceabilityMarkStartTracer tracer; + + ZCollectedHeap::heap()->increment_total_collections(true /* full */); + + ZHeap::heap()->mark_start(); + return true; + } +}; + +class ZMarkEndClosure : public ZOperationClosure { +public: + virtual const char* name() const { + return "ZMarkEnd"; + } + + virtual bool do_operation() { + ZStatTimer timer(ZPhasePauseMarkEnd); + ZServiceabilityMarkEndTracer tracer; + + return ZHeap::heap()->mark_end(); + } +}; + +class ZRelocateStartClosure : public ZOperationClosure { +public: + virtual const char* name() const { + return "ZRelocateStart"; + } + + virtual bool needs_inactive_gc_locker() const { + return true; + } + + virtual bool do_operation() { + ZStatTimer timer(ZPhasePauseRelocateStart); + ZServiceabilityRelocateStartTracer tracer; + + ZHeap::heap()->relocate_start(); + return true; + } +}; + +ZDriver::ZDriver() : + _gc_cycle_port(), + _gc_locker_port() { + set_name("ZDriver"); + create_and_start(); +} + +bool ZDriver::vm_operation(ZOperationClosure* cl) { + for (;;) { + VM_ZOperation op(cl); + VMThread::execute(&op); + if (op.gc_locked()) { + // Wait for GC to become unlocked and restart the VM operation + ZStatTimer timer(ZCriticalPhaseGCLockerStall); + _gc_locker_port.wait(); + continue; + } + + // Notify VM operation completed + _gc_locker_port.ack(); + + return op.success(); + } +} + +void ZDriver::collect(GCCause::Cause cause) { + switch (cause) { + case GCCause::_wb_young_gc: + case GCCause::_wb_conc_mark: + case GCCause::_wb_full_gc: + case GCCause::_dcmd_gc_run: + case GCCause::_java_lang_system_gc: + case GCCause::_full_gc_alot: + case GCCause::_scavenge_alot: + case GCCause::_jvmti_force_gc: + case GCCause::_metadata_GC_clear_soft_refs: + // Start synchronous GC + _gc_cycle_port.send_sync(cause); + break; + + case GCCause::_z_timer: + case GCCause::_z_warmup: + case GCCause::_z_allocation_rate: + case GCCause::_z_allocation_stall: + case GCCause::_z_proactive: + case GCCause::_metadata_GC_threshold: + // Start asynchronous GC + _gc_cycle_port.send_async(cause); + break; + + case GCCause::_gc_locker: + // Restart VM operation previously blocked by the GC locker + _gc_locker_port.signal(); + break; + + default: + // Other causes not supported + fatal("Unsupported GC cause (%s)", GCCause::to_string(cause)); + break; + } +} + +GCCause::Cause ZDriver::start_gc_cycle() { + // Wait for GC request + return _gc_cycle_port.receive(); +} + +class ZSoftReferencePolicyScope : public StackObj { +private: + bool should_clear_soft_reference(GCCause::Cause cause) const { + const bool clear = ZCollectedHeap::heap()->soft_ref_policy()->should_clear_all_soft_refs(); + + // Clear all soft reference if the policy says so, or if + // the GC cause indicates that we're running low on memory. + return clear || + cause == GCCause::_z_allocation_stall || + cause == GCCause::_metadata_GC_clear_soft_refs; + } + + void clear_should_clear_soft_reference() const { + ZCollectedHeap::heap()->soft_ref_policy()->set_should_clear_all_soft_refs(false); + } + +public: + ZSoftReferencePolicyScope(GCCause::Cause cause) { + const bool clear = should_clear_soft_reference(cause); + ZHeap::heap()->set_soft_reference_policy(clear); + clear_should_clear_soft_reference(); + } + + ~ZSoftReferencePolicyScope() { + Universe::update_heap_info_at_gc(); + } +}; + +class ZDriverCycleScope : public StackObj { +private: + GCIdMark _gc_id; + GCCauseSetter _gc_cause_setter; + ZSoftReferencePolicyScope _soft_ref_policy; + ZStatTimer _timer; + + bool should_boost_worker_threads(GCCause::Cause cause) const { + return cause == GCCause::_java_lang_system_gc || + cause == GCCause::_z_allocation_stall; + } + +public: + ZDriverCycleScope(GCCause::Cause cause) : + _gc_id(), + _gc_cause_setter(ZCollectedHeap::heap(), cause), + _soft_ref_policy(cause), + _timer(ZPhaseCycle) { + // Update statistics + ZStatCycle::at_start(); + + // Set boost mode + const bool boost = should_boost_worker_threads(cause); + ZHeap::heap()->set_boost_worker_threads(boost); + } + + ~ZDriverCycleScope() { + // Calculate boost factor + const double boost_factor = (double)ZHeap::heap()->nconcurrent_worker_threads() / + (double)ZHeap::heap()->nconcurrent_no_boost_worker_threads(); + + // Update statistics + ZStatCycle::at_end(boost_factor); + } +}; + +void ZDriver::run_gc_cycle(GCCause::Cause cause) { + ZDriverCycleScope scope(cause); + + // Phase 1: Pause Mark Start + { + ZMarkStartClosure cl; + vm_operation(&cl); + } + + // Phase 2: Concurrent Mark + { + ZStatTimer timer(ZPhaseConcurrentMark); + ZHeap::heap()->mark(); + } + + // Phase 3: Pause Mark End + { + ZMarkEndClosure cl; + while (!vm_operation(&cl)) { + // Phase 3.5: Concurrent Mark Continue + ZStatTimer timer(ZPhaseConcurrentMarkContinue); + ZHeap::heap()->mark(); + } + } + + // Phase 4: Concurrent Process Non-Strong References + { + ZStatTimer timer(ZPhaseConcurrentProcessNonStrongReferences); + ZHeap::heap()->process_non_strong_references(); + } + + // Phase 5: Concurrent Reset Relocation Set + { + ZStatTimer timer(ZPhaseConcurrentResetRelocationSet); + ZHeap::heap()->reset_relocation_set(); + } + + // Phase 6: Concurrent Destroy Detached Pages + { + ZStatTimer timer(ZPhaseConcurrentDestroyDetachedPages); + ZHeap::heap()->destroy_detached_pages(); + } + + // Phase 7: Concurrent Select Relocation Set + { + ZStatTimer timer(ZPhaseConcurrentSelectRelocationSet); + ZHeap::heap()->select_relocation_set(); + } + + // Phase 8: Prepare Relocation Set + { + ZStatTimer timer(ZPhaseConcurrentPrepareRelocationSet); + ZHeap::heap()->prepare_relocation_set(); + } + + // Phase 9: Pause Relocate Start + { + ZRelocateStartClosure cl; + vm_operation(&cl); + } + + // Phase 10: Concurrent Relocate + { + ZStatTimer timer(ZPhaseConcurrentRelocated); + ZHeap::heap()->relocate(); + } +} + +void ZDriver::end_gc_cycle() { + // Notify GC cycle completed + _gc_cycle_port.ack(); + + // Check for out of memory condition + ZHeap::heap()->check_out_of_memory(); +} + +void ZDriver::run_service() { + // Main loop + while (!should_terminate()) { + const GCCause::Cause cause = start_gc_cycle(); + if (cause != GCCause::_no_gc) { + run_gc_cycle(cause); + end_gc_cycle(); + } + } +} + +void ZDriver::stop_service() { + _gc_cycle_port.send_async(GCCause::_no_gc); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zDriver.hpp 2018-06-01 22:30:29.801414351 +0200 @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZDRIVER_HPP +#define SHARE_GC_Z_ZDRIVER_HPP + +#include "gc/shared/concurrentGCThread.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/z/zMessagePort.hpp" + +class ZOperationClosure; + +class ZDriver : public ConcurrentGCThread { +private: + ZMessagePort _gc_cycle_port; + ZRendezvousPort _gc_locker_port; + + bool vm_operation(ZOperationClosure* cl); + + GCCause::Cause start_gc_cycle(); + void run_gc_cycle(GCCause::Cause cause); + void end_gc_cycle(); + +protected: + virtual void run_service(); + virtual void stop_service(); + +public: + ZDriver(); + + void collect(GCCause::Cause cause); +}; + +#endif // SHARE_GC_Z_ZDRIVER_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zErrno.cpp 2018-06-01 22:30:30.185430927 +0200 @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zErrno.hpp" + +#include +#include + +ZErrno::ZErrno() : + _error(errno) {} + +ZErrno::ZErrno(int error) : + _error(error) {} + +ZErrno::operator bool() const { + return _error != 0; +} + +bool ZErrno::operator==(int error) const { + return _error == error; +} + +bool ZErrno::operator!=(int error) const { + return _error != error; +} + +const char* ZErrno::to_string() const { + return strerror(_error); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zErrno.hpp 2018-06-01 22:30:30.569447502 +0200 @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZERRNO_HPP +#define SHARE_GC_Z_ZERRNO_HPP + +#include "memory/allocation.hpp" + +class ZErrno : public StackObj { +private: + const int _error; + +public: + ZErrno(); + ZErrno(int error); + + operator bool() const; + bool operator==(int error) const; + bool operator!=(int error) const; + const char* to_string() const; +}; + +#endif // SHARE_GC_Z_ZERRNO_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zFlags.hpp 2018-06-01 22:30:30.939463474 +0200 @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZFLAGS_HPP +#define SHARE_GC_Z_ZFLAGS_HPP + +#define GC_Z_FLAGS(develop, \ + develop_pd, \ + product, \ + product_pd, \ + diagnostic, \ + diagnostic_pd, \ + experimental, \ + notproduct, \ + manageable, \ + product_rw, \ + lp64_product, \ + range, \ + constraint, \ + writeable) \ + \ + product(ccstr, ZPath, NULL, \ + "Filesystem path for Java heap backing storage " \ + "(must be a tmpfs or a hugetlbfs filesystem)") \ + \ + product(double, ZAllocationSpikeTolerance, 2.0, \ + "Allocation spike tolerance factor") \ + \ + product(double, ZFragmentationLimit, 25.0, \ + "Maximum allowed heap fragmentation") \ + \ + product(bool, ZStallOnOutOfMemory, true, \ + "Allow Java threads to stall and wait for GC to complete " \ + "instead of immediately throwing an OutOfMemoryError") \ + \ + product(size_t, ZMarkStacksMax, NOT_LP64(512*M) LP64_ONLY(8*G), \ + "Maximum number of bytes allocated for marking stacks") \ + range(32*M, NOT_LP64(512*M) LP64_ONLY(1024*G)) \ + \ + product(uint, ZCollectionInterval, 0, \ + "Force GC at a fixed time interval (in seconds)") \ + \ + product(uint, ZStatisticsInterval, 10, \ + "Time between statistics print outs (in seconds)") \ + range(1, (uint)-1) \ + \ + diagnostic(bool, ZStatisticsForceTrace, false, \ + "Force tracing of ZStats") \ + \ + diagnostic(bool, ZUnmapBadViews, false, \ + "Unmap bad (inactive) heap views") \ + \ + diagnostic(bool, ZVerifyMarking, false, \ + "Verify marking stacks") \ + \ + diagnostic(bool, ZVerifyForwarding, false, \ + "Verify forwarding tables") \ + \ + diagnostic(bool, ZSymbolTableUnloading, false, \ + "Unload unused VM symbols") \ + \ + diagnostic(bool, ZWeakRoots, true, \ + "Treat JNI WeakGlobalRefs and StringTable as weak roots") \ + \ + diagnostic(bool, ZConcurrentVMWeakHandles, true, \ + "Clean VM WeakHandles concurrently") \ + \ + diagnostic(bool, ZConcurrentJNIWeakGlobalHandles, true, \ + "Clean JNI WeakGlobalRefs concurrently") \ + \ + diagnostic(bool, ZOptimizeLoadBarriers, true, \ + "Apply load barrier optimizations") \ + \ + develop(bool, ZVerifyLoadBarriers, false, \ + "Verify that reference loads are followed by barriers") + +#endif // SHARE_GC_Z_ZFLAGS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zForwardingTable.cpp 2018-06-01 22:30:31.309479445 +0200 @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zForwardingTable.inline.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "memory/allocation.inline.hpp" +#include "utilities/debug.hpp" + +void ZForwardingTable::setup(size_t live_objects) { + assert(is_null(), "Should be empty"); + assert(live_objects > 0, "Invalid size"); + + // Allocate table for linear probing. The size of the table must be + // a power of two to allow for quick and inexpensive indexing/masking. + // The table is sized to have a load factor of 50%, i.e. sized to have + // double the number of entries actuallly inserted. + _size = ZUtils::round_up_power_of_2(live_objects * 2); + _table = MallocArrayAllocator::allocate(_size, mtGC); + + // Clear table + memset(_table, ZForwardingTableEntry::empty(), _size * sizeof(ZForwardingTableEntry)); +} + +void ZForwardingTable::reset() { + // Free table + MallocArrayAllocator::free(_table); + _table = NULL; + _size = 0; +} + +void ZForwardingTable::verify(size_t object_max_count, size_t live_objects) const { + size_t count = 0; + + for (size_t i = 0; i < _size; i++) { + const ZForwardingTableEntry entry = _table[i]; + if (entry.is_empty()) { + // Skip empty entries + continue; + } + + // Check from index + guarantee(entry.from_index() < object_max_count, "Invalid from index"); + + // Check for duplicates + for (size_t j = i + 1; j < _size; j++) { + const ZForwardingTableEntry other = _table[j]; + guarantee(entry.from_index() != other.from_index(), "Duplicate from"); + guarantee(entry.to_offset() != other.to_offset(), "Duplicate to"); + } + + count++; + } + + // Check number of non-null entries + guarantee(live_objects == count, "Count mismatch"); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zForwardingTable.hpp 2018-06-01 22:30:31.694496064 +0200 @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZFORWARDING_HPP +#define SHARE_GC_Z_ZFORWARDING_HPP + +#include "gc/z/zForwardingTableEntry.hpp" +#include "memory/allocation.hpp" + +typedef size_t ZForwardingTableCursor; + +class ZForwardingTable { + friend class VMStructs; + friend class ZForwardingTableTest; + +private: + ZForwardingTableEntry* _table; + size_t _size; + + ZForwardingTableEntry at(ZForwardingTableCursor* cursor) const; + ZForwardingTableEntry first(uintptr_t from_index, ZForwardingTableCursor* cursor) const; + ZForwardingTableEntry next(ZForwardingTableCursor* cursor) const; + +public: + ZForwardingTable(); + ~ZForwardingTable(); + + bool is_null() const; + void setup(size_t live_objects); + void reset(); + + ZForwardingTableEntry find(uintptr_t from_index) const; + ZForwardingTableEntry find(uintptr_t from_index, ZForwardingTableCursor* cursor) const; + uintptr_t insert(uintptr_t from_index, uintptr_t to_offset, ZForwardingTableCursor* cursor); + + void verify(size_t object_max_count, size_t live_objects) const; +}; + +#endif // SHARE_GC_Z_ZFORWARDING_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zForwardingTable.inline.hpp 2018-06-01 22:30:32.070512295 +0200 @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZFORWARDING_INLINE_HPP +#define SHARE_GC_Z_ZFORWARDING_INLINE_HPP + +#include "gc/z/zForwardingTable.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHash.inline.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" + +inline ZForwardingTable::ZForwardingTable() : + _table(NULL), + _size(0) {} + +inline ZForwardingTable::~ZForwardingTable() { + assert(is_null(), "Should be empty"); +} + +inline ZForwardingTableEntry ZForwardingTable::at(ZForwardingTableCursor* cursor) const { + return _table[*cursor]; +} + +inline ZForwardingTableEntry ZForwardingTable::first(uintptr_t from_index, ZForwardingTableCursor* cursor) const { + const size_t mask = _size - 1; + const size_t hash = ZHash::uint32_to_uint32((uint32_t)from_index); + *cursor = hash & mask; + return at(cursor); +} + +inline ZForwardingTableEntry ZForwardingTable::next(ZForwardingTableCursor* cursor) const { + const size_t mask = _size - 1; + *cursor = (*cursor + 1) & mask; + return at(cursor); +} + +inline bool ZForwardingTable::is_null() const { + return _table == NULL; +} + +inline ZForwardingTableEntry ZForwardingTable::find(uintptr_t from_index) const { + ZForwardingTableCursor dummy; + return find(from_index, &dummy); +} + +inline ZForwardingTableEntry ZForwardingTable::find(uintptr_t from_index, ZForwardingTableCursor* cursor) const { + // Reading entries in the table races with the atomic cas done for + // insertion into the table. This is safe because each entry is at + // most updated once (from -1 to something else). + ZForwardingTableEntry entry = first(from_index, cursor); + while (!entry.is_empty()) { + if (entry.from_index() == from_index) { + // Match found, return matching entry + return entry; + } + + entry = next(cursor); + } + + // Match not found, return empty entry + return entry; +} + +inline uintptr_t ZForwardingTable::insert(uintptr_t from_index, uintptr_t to_offset, ZForwardingTableCursor* cursor) { + const ZForwardingTableEntry new_entry(from_index, to_offset); + const ZForwardingTableEntry old_entry; // empty + + for (;;) { + const ZForwardingTableEntry prev_entry = Atomic::cmpxchg(new_entry, _table + *cursor, old_entry); + if (prev_entry.is_empty()) { + // Success + return to_offset; + } + + // Find next empty or matching entry + ZForwardingTableEntry entry = at(cursor); + while (!entry.is_empty()) { + if (entry.from_index() == from_index) { + // Match found, return already inserted address + return entry.to_offset(); + } + + entry = next(cursor); + } + } +} + +#endif // SHARE_GC_Z_ZFORWARDING_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zForwardingTableEntry.hpp 2018-06-01 22:30:32.457529000 +0200 @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZFORWARDINGTABLEENTRY_HPP +#define SHARE_GC_Z_ZFORWARDINGTABLEENTRY_HPP + +#include "gc/z/zBitField.hpp" +#include "memory/allocation.hpp" +#include "metaprogramming/primitiveConversions.hpp" + +// +// Forwarding table entry layout +// ----------------------------- +// +// 6 4 4 0 +// 3 2 1 0 +// +------------------------+-----------------------------------------------+ +// |11111111 11111111 111111|11 11111111 11111111 11111111 11111111 11111111| +// +------------------------+-----------------------------------------------+ +// | | +// | * 41-0 To Object Offset (42-bits) +// | +// * 63-42 From Object Index (22-bits) +// + +class ZForwardingTableEntry { + friend struct PrimitiveConversions; + +private: + typedef ZBitField field_to_offset; + typedef ZBitField field_from_index; + + uint64_t _entry; + +public: + ZForwardingTableEntry() : + _entry(empty()) {} + + ZForwardingTableEntry(size_t from_index, size_t to_offset) : + _entry(field_from_index::encode(from_index) | + field_to_offset::encode(to_offset)) {} + + static uintptr_t empty() { + return (uintptr_t)-1; + } + + bool is_empty() const { + return _entry == empty(); + } + + size_t to_offset() const { + return field_to_offset::decode(_entry); + } + + size_t from_index() const { + return field_from_index::decode(_entry); + } +}; + +// Needed to allow atomic operations on ZForwardingTableEntry +template <> +struct PrimitiveConversions::Translate : public TrueType { + typedef ZForwardingTableEntry Value; + typedef uint64_t Decayed; + + static Decayed decay(Value v) { + return v._entry; + } + + static Value recover(Decayed d) { + ZForwardingTableEntry entry; + entry._entry = d; + return entry; + } +}; + +#endif // SHARE_GC_Z_ZFORWARDINGTABLEENTRY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zFuture.hpp 2018-06-01 22:30:32.832545187 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZFUTURE_HPP +#define SHARE_GC_Z_ZFUTURE_HPP + +#include "memory/allocation.hpp" +#include "runtime/semaphore.hpp" + +template +class ZFuture { +private: + Semaphore _sema; + T _value; + +public: + void set(T value); + T get(); +}; + +#endif // SHARE_GC_Z_ZFUTURE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zFuture.inline.hpp 2018-06-01 22:30:33.217561806 +0200 @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZFUTURE_INLINE_HPP +#define SHARE_GC_Z_ZFUTURE_INLINE_HPP + +#include "gc/z/zFuture.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "runtime/semaphore.inline.hpp" +#include "runtime/thread.hpp" + +template +inline void ZFuture::set(T value) { + // Set value + _value = value; + + // Notify waiter + _sema.signal(); +} + +template +inline T ZFuture::get() { + // Wait for notification + Thread* const thread = Thread::current(); + if (thread->is_Java_thread()) { + _sema.wait_with_safepoint_check((JavaThread*)thread); + } else { + _sema.wait(); + } + + // Return value + return _value; +} + +#endif // SHARE_GC_Z_ZFUTURE_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zGlobals.cpp 2018-06-01 22:30:33.599578295 +0200 @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zGlobals.hpp" + +uint32_t ZGlobalPhase = ZPhaseRelocate; +uint32_t ZGlobalSeqNum = 1; + +const int& ZObjectAlignmentSmallShift = LogMinObjAlignmentInBytes; +const int& ZObjectAlignmentSmall = MinObjAlignmentInBytes; + +uintptr_t ZAddressGoodMask; +uintptr_t ZAddressBadMask = 0; +uintptr_t ZAddressWeakBadMask; + +uintptr_t ZAddressMetadataMarked; --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zGlobals.hpp 2018-06-01 22:30:33.984594914 +0200 @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZGLOBALS_HPP +#define SHARE_GC_Z_ZGLOBALS_HPP + +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" +#include OS_CPU_HEADER(gc/z/zGlobals) + +// Collector name +const char* const ZGCName = "The Z Garbage Collector"; + +// Global phase state +extern uint32_t ZGlobalPhase; +const uint32_t ZPhaseMark = 0; +const uint32_t ZPhaseMarkCompleted = 1; +const uint32_t ZPhaseRelocate = 2; + +// Global sequence number +extern uint32_t ZGlobalSeqNum; + +// Page types +const uint8_t ZPageTypeSmall = 0; +const uint8_t ZPageTypeMedium = 1; +const uint8_t ZPageTypeLarge = 2; + +// Page size shifts +const size_t ZPageSizeSmallShift = ZPlatformPageSizeSmallShift; +const size_t ZPageSizeMediumShift = ZPageSizeSmallShift + 4; +const size_t ZPageSizeMinShift = ZPageSizeSmallShift; + +// Page sizes +const size_t ZPageSizeSmall = (size_t)1 << ZPageSizeSmallShift; +const size_t ZPageSizeMedium = (size_t)1 << ZPageSizeMediumShift; +const size_t ZPageSizeMin = (size_t)1 << ZPageSizeMinShift; + +// Object size limits +const size_t ZObjectSizeLimitSmall = (ZPageSizeSmall / 8); // Allow 12.5% waste +const size_t ZObjectSizeLimitMedium = (ZPageSizeMedium / 8); // Allow 12.5% waste + +// Object alignment shifts +extern const int& ZObjectAlignmentSmallShift; +const int ZObjectAlignmentMediumShift = ZPageSizeMediumShift - 13; // 8192 objects per page +const int ZObjectAlignmentLargeShift = ZPageSizeSmallShift; + +// Object alignments +extern const int& ZObjectAlignmentSmall; +const int ZObjectAlignmentMedium = 1 << ZObjectAlignmentMediumShift; +const int ZObjectAlignmentLarge = 1 << ZObjectAlignmentLargeShift; + +// Pointer part of address +const uintptr_t ZAddressOffsetShift = 0; +const uintptr_t ZAddressOffsetBits = ZPlatformAddressOffsetBits; +const uintptr_t ZAddressOffsetMask = (((uintptr_t)1 << ZAddressOffsetBits) - 1) << ZAddressOffsetShift; +const size_t ZAddressOffsetMax = (uintptr_t)1 << ZAddressOffsetBits; + +// Metadata part of address +const uintptr_t ZAddressMetadataShift = ZPlatformAddressMetadataShift; +const uintptr_t ZAddressMetadataBits = 4; +const uintptr_t ZAddressMetadataMask = (((uintptr_t)1 << ZAddressMetadataBits) - 1) << ZAddressMetadataShift; + +// Metadata types +const uintptr_t ZAddressMetadataMarked0 = (uintptr_t)1 << (ZAddressMetadataShift + 0); +const uintptr_t ZAddressMetadataMarked1 = (uintptr_t)1 << (ZAddressMetadataShift + 1); +const uintptr_t ZAddressMetadataRemapped = (uintptr_t)1 << (ZAddressMetadataShift + 2); +const uintptr_t ZAddressMetadataFinalizable = (uintptr_t)1 << (ZAddressMetadataShift + 3); + +// Address space start/end/size +const uintptr_t ZAddressSpaceStart = ZPlatformAddressSpaceStart; +const uintptr_t ZAddressSpaceSize = ZPlatformAddressSpaceSize; +const uintptr_t ZAddressSpaceEnd = ZAddressSpaceStart + ZAddressSpaceSize; + +// Cache line size +const size_t ZCacheLineSize = ZPlatformCacheLineSize; + +// Reserved start/end +uintptr_t ZAddressReservedStart(); +uintptr_t ZAddressReservedEnd(); + +// +// Good/Bad mask states +// -------------------- +// +// GoodMask BadMask WeakGoodMask WeakBadMask +// -------------------------------------------------------------- +// Marked0 001 110 101 010 +// Marked1 010 101 110 001 +// Remapped 100 011 100 011 +// + +// Good/bad masks +extern uintptr_t ZAddressGoodMask; +extern uintptr_t ZAddressBadMask; +extern uintptr_t ZAddressWeakBadMask; + +// Marked state +extern uintptr_t ZAddressMetadataMarked; + +// Address space for mark stack allocations +const size_t ZMarkStackSpaceSizeShift = 40; // 1TB +const size_t ZMarkStackSpaceSize = (size_t)1 << ZMarkStackSpaceSizeShift; +const uintptr_t ZMarkStackSpaceStart = ZAddressSpaceEnd + ZMarkStackSpaceSize; +const uintptr_t ZMarkStackSpaceEnd = ZMarkStackSpaceStart + ZMarkStackSpaceSize; +const size_t ZMarkStackSpaceExpandSize = (size_t)1 << 25; // 32M + +// Mark stack and magazine sizes +const size_t ZMarkStackSizeShift = 11; // 2K +const size_t ZMarkStackSize = (size_t)1 << ZMarkStackSizeShift; +const size_t ZMarkStackHeaderSize = (size_t)1 << 4; // 16B +const size_t ZMarkStackSlots = (ZMarkStackSize - ZMarkStackHeaderSize) / sizeof(uintptr_t); +const size_t ZMarkStackMagazineSize = (size_t)1 << 15; // 32K +const size_t ZMarkStackMagazineSlots = (ZMarkStackMagazineSize / ZMarkStackSize) - 1; + +// Mark stripe size +const size_t ZMarkStripeShift = ZPageSizeMinShift; + +// Max number of mark stripes +const size_t ZMarkStripesMax = 16; // Must be a power of two + +// Mark cache size +const size_t ZMarkCacheSize = 1024; // Must be a power of two + +// Partial array minimum size +const size_t ZMarkPartialArrayMinSizeShift = 12; // 4K +const size_t ZMarkPartialArrayMinSize = (size_t)1 << ZMarkPartialArrayMinSizeShift; + +// Max number of proactive/terminate flush attempts +const size_t ZMarkProactiveFlushMax = 10; +const size_t ZMarkTerminateFlushMax = 3; + +// Try complete mark timeout +const uint64_t ZMarkCompleteTimeout = 1; // ms + +#endif // SHARE_GC_Z_ZGLOBALS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zHash.hpp 2018-06-01 22:30:34.381612051 +0200 @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZHASH_HPP +#define SHARE_GC_Z_ZHASH_HPP + +#include "memory/allocation.hpp" +#include "utilities/globalDefinitions.hpp" + +class ZHash : public AllStatic { +public: + static uint32_t uint32_to_uint32(uint32_t key); + static uint32_t address_to_uint32(uintptr_t key); +}; + +#endif // SHARE_GC_Z_ZHASH_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zHash.inline.hpp 2018-06-01 22:30:34.767628713 +0200 @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZHASH_INLINE_HPP +#define SHARE_GC_Z_ZHASH_INLINE_HPP + +#include "gc/z/zHash.hpp" + +inline uint32_t ZHash::uint32_to_uint32(uint32_t key) { + key = ~key + (key << 15); + key = key ^ (key >> 12); + key = key + (key << 2); + key = key ^ (key >> 4); + key = key * 2057; + key = key ^ (key >> 16); + return key; +} + +inline uint32_t ZHash::address_to_uint32(uintptr_t key) { + return uint32_to_uint32((uint32_t)(key >> 3)); +} + +#endif // SHARE_GC_Z_ZHASH_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zHeap.cpp 2018-06-01 22:30:35.147645116 +0200 @@ -0,0 +1,583 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/oopStorage.hpp" +#include "gc/z/zAddress.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zHeapIterator.hpp" +#include "gc/z/zList.inline.hpp" +#include "gc/z/zLock.inline.hpp" +#include "gc/z/zMark.inline.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zPageTable.inline.hpp" +#include "gc/z/zRelocationSet.inline.hpp" +#include "gc/z/zResurrection.hpp" +#include "gc/z/zRootsIterator.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zThread.hpp" +#include "gc/z/zTracer.inline.hpp" +#include "gc/z/zVirtualMemory.inline.hpp" +#include "gc/z/zWorkers.inline.hpp" +#include "logging/log.hpp" +#include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/thread.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes); +static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes); +static const ZStatSampler ZSamplerHeapUsedBeforeRelocation("Memory", "Heap Used Before Relocation", ZStatUnitBytes); +static const ZStatSampler ZSamplerHeapUsedAfterRelocation("Memory", "Heap Used After Relocation", ZStatUnitBytes); +static const ZStatCounter ZCounterUndoPageAllocation("Memory", "Undo Page Allocation", ZStatUnitOpsPerSecond); +static const ZStatCounter ZCounterOutOfMemory("Memory", "Out Of Memory", ZStatUnitOpsPerSecond); + +ZHeap* ZHeap::_heap = NULL; + +ZHeap::ZHeap() : + _workers(), + _object_allocator(_workers.nworkers()), + _page_allocator(heap_min_size(), heap_max_size(), heap_max_reserve_size()), + _pagetable(), + _mark(&_workers, &_pagetable), + _reference_processor(&_workers), + _weak_roots_processor(&_workers), + _relocate(&_workers), + _relocation_set(), + _serviceability(heap_min_size(), heap_max_size()) { + // Install global heap instance + assert(_heap == NULL, "Already initialized"); + _heap = this; + + // Update statistics + ZStatHeap::set_at_initialize(heap_max_size(), heap_max_reserve_size()); +} + +size_t ZHeap::heap_min_size() const { + const size_t aligned_min_size = align_up(InitialHeapSize, ZPageSizeMin); + return MIN2(aligned_min_size, heap_max_size()); +} + +size_t ZHeap::heap_max_size() const { + const size_t aligned_max_size = align_up(MaxHeapSize, ZPageSizeMin); + return MIN2(aligned_max_size, ZAddressOffsetMax); +} + +size_t ZHeap::heap_max_reserve_size() const { + // Reserve one small page per worker plus one shared medium page. This is still just + // an estimate and doesn't guarantee that we can't run out of memory during relocation. + const size_t max_reserve_size = (_workers.nworkers() * ZPageSizeSmall) + ZPageSizeMedium; + return MIN2(max_reserve_size, heap_max_size()); +} + +bool ZHeap::is_initialized() const { + return _page_allocator.is_initialized(); +} + +size_t ZHeap::min_capacity() const { + return heap_min_size(); +} + +size_t ZHeap::max_capacity() const { + return _page_allocator.max_capacity(); +} + +size_t ZHeap::capacity() const { + return _page_allocator.capacity(); +} + +size_t ZHeap::max_reserve() const { + return _page_allocator.max_reserve(); +} + +size_t ZHeap::used_high() const { + return _page_allocator.used_high(); +} + +size_t ZHeap::used_low() const { + return _page_allocator.used_low(); +} + +size_t ZHeap::used() const { + return _page_allocator.used(); +} + +size_t ZHeap::allocated() const { + return _page_allocator.allocated(); +} + +size_t ZHeap::reclaimed() const { + return _page_allocator.reclaimed(); +} + +size_t ZHeap::tlab_capacity() const { + return capacity(); +} + +size_t ZHeap::tlab_used() const { + return _object_allocator.used(); +} + +size_t ZHeap::max_tlab_size() const { + return ZObjectSizeLimitSmall; +} + +size_t ZHeap::unsafe_max_tlab_alloc() const { + size_t size = _object_allocator.remaining(); + + if (size < MinTLABSize) { + // The remaining space in the allocator is not enough to + // fit the smallest possible TLAB. This means that the next + // TLAB allocation will force the allocator to get a new + // backing page anyway, which in turn means that we can then + // fit the larges possible TLAB. + size = max_tlab_size(); + } + + return MIN2(size, max_tlab_size()); +} + +bool ZHeap::is_in(uintptr_t addr) const { + if (addr < ZAddressReservedStart() || addr >= ZAddressReservedEnd()) { + return false; + } + + const ZPage* const page = _pagetable.get(addr); + if (page != NULL) { + return page->is_in(addr); + } + + return false; +} + +uintptr_t ZHeap::block_start(uintptr_t addr) const { + const ZPage* const page = _pagetable.get(addr); + return page->block_start(addr); +} + +size_t ZHeap::block_size(uintptr_t addr) const { + const ZPage* const page = _pagetable.get(addr); + return page->block_size(addr); +} + +bool ZHeap::block_is_obj(uintptr_t addr) const { + const ZPage* const page = _pagetable.get(addr); + return page->block_is_obj(addr); +} + +uint ZHeap::nconcurrent_worker_threads() const { + return _workers.nconcurrent(); +} + +uint ZHeap::nconcurrent_no_boost_worker_threads() const { + return _workers.nconcurrent_no_boost(); +} + +void ZHeap::set_boost_worker_threads(bool boost) { + _workers.set_boost(boost); +} + +void ZHeap::worker_threads_do(ThreadClosure* tc) const { + _workers.threads_do(tc); +} + +void ZHeap::print_worker_threads_on(outputStream* st) const { + _workers.print_threads_on(st); +} + +void ZHeap::out_of_memory() { + ResourceMark rm; + + ZStatInc(ZCounterOutOfMemory); + log_info(gc)("Out Of Memory (%s)", Thread::current()->name()); +} + +ZPage* ZHeap::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) { + ZPage* const page = _page_allocator.alloc_page(type, size, flags); + if (page != NULL) { + // Update pagetable + _pagetable.insert(page); + } + + return page; +} + +void ZHeap::undo_alloc_page(ZPage* page) { + assert(page->is_allocating(), "Invalid page state"); + + ZStatInc(ZCounterUndoPageAllocation); + log_trace(gc)("Undo page allocation, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT ", size: " SIZE_FORMAT, + ZThread::id(), ZThread::name(), p2i(page), page->size()); + + release_page(page, false /* reclaimed */); +} + +bool ZHeap::retain_page(ZPage* page) { + return page->inc_refcount(); +} + +void ZHeap::release_page(ZPage* page, bool reclaimed) { + if (page->dec_refcount()) { + _page_allocator.free_page(page, reclaimed); + } +} + +void ZHeap::flip_views() { + // For debugging only + if (ZUnmapBadViews) { + // Flip pages + ZPageTableIterator iter(&_pagetable); + for (ZPage* page; iter.next(&page);) { + if (!page->is_detached()) { + _page_allocator.flip_page(page); + } + } + + // Flip pre-mapped memory + _page_allocator.flip_pre_mapped(); + } +} + +void ZHeap::mark_start() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + + // Update statistics + ZStatSample(ZSamplerHeapUsedBeforeMark, used()); + + // Retire TLABs + _object_allocator.retire_tlabs(); + + // Flip address view + ZAddressMasks::flip_to_marked(); + flip_views(); + + // Reset allocated/reclaimed/used statistics + _page_allocator.reset_statistics(); + + // Reset encountered/dropped/enqueued statistics + _reference_processor.reset_statistics(); + + // Enter mark phase + ZGlobalPhase = ZPhaseMark; + + // Reset marking information and mark roots + _mark.start(); + + // Update statistics + ZStatHeap::set_at_mark_start(capacity(), used()); +} + +void ZHeap::mark() { + _mark.mark(); +} + +void ZHeap::mark_flush_and_free(Thread* thread) { + _mark.flush_and_free(thread); +} + +class ZFixupPartialLoadsTask : public ZTask { +private: + ZThreadRootsIterator _thread_roots; + +public: + ZFixupPartialLoadsTask() : + ZTask("ZFixupPartialLoadsTask"), + _thread_roots() {} + + virtual void work() { + ZMarkRootOopClosure cl; + _thread_roots.oops_do(&cl); + } +}; + +void ZHeap::fixup_partial_loads() { + ZFixupPartialLoadsTask task; + _workers.run_parallel(&task); +} + +bool ZHeap::mark_end() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + + // C2 can generate code where a safepoint poll is inserted + // between a load and the associated load barrier. To handle + // this case we need to rescan the thread stack here to make + // sure such oops are marked. + fixup_partial_loads(); + + // Try end marking + if (!_mark.end()) { + // Marking not completed, continue concurrent mark + return false; + } + + // Enter mark completed phase + ZGlobalPhase = ZPhaseMarkCompleted; + + // Resize metaspace + MetaspaceGC::compute_new_size(); + + // Update statistics + ZStatSample(ZSamplerHeapUsedAfterMark, used()); + ZStatHeap::set_at_mark_end(capacity(), allocated(), used()); + + // Block resurrection of weak/phantom references + ZResurrection::block(); + + // Process weak roots + _weak_roots_processor.process_weak_roots(); + + // Verification + if (VerifyBeforeGC || VerifyDuringGC || VerifyAfterGC) { + Universe::verify(); + } + + return true; +} + +void ZHeap::set_soft_reference_policy(bool clear) { + _reference_processor.set_soft_reference_policy(clear); +} + +void ZHeap::process_non_strong_references() { + // Process Soft/Weak/Final/PhantomReferences + _reference_processor.process_references(); + + // Process concurrent weak roots + _weak_roots_processor.process_concurrent_weak_roots(); + + // Unblock resurrection of weak/phantom references + ZResurrection::unblock(); + + // Enqueue Soft/Weak/Final/PhantomReferences. Note that this + // must be done after unblocking resurrection. Otherwise the + // Finalizer thread could call Reference.get() on the Finalizers + // that were just enqueued, which would incorrectly return null + // during the resurrection block window, since such referents + // are only Finalizable marked. + _reference_processor.enqueue_references(); +} + +void ZHeap::destroy_detached_pages() { + ZList list; + + _page_allocator.flush_detached_pages(&list); + + for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) { + // Remove pagetable entry + _pagetable.remove(page); + + // Delete the page + _page_allocator.destroy_page(page); + } +} + +void ZHeap::select_relocation_set() { + // Register relocatable pages with selector + ZRelocationSetSelector selector; + ZPageTableIterator iter(&_pagetable); + for (ZPage* page; iter.next(&page);) { + if (!page->is_relocatable()) { + // Not relocatable, don't register + continue; + } + + if (page->is_marked()) { + // Register live page + selector.register_live_page(page); + } else { + // Register garbage page + selector.register_garbage_page(page); + + // Reclaim page immediately + release_page(page, true /* reclaimed */); + } + } + + // Select pages to relocate + selector.select(&_relocation_set); + + // Update statistics + ZStatRelocation::set_at_select_relocation_set(selector.relocating()); + ZStatHeap::set_at_select_relocation_set(selector.live(), + selector.garbage(), + reclaimed()); +} + +void ZHeap::prepare_relocation_set() { + ZRelocationSetIterator iter(&_relocation_set); + for (ZPage* page; iter.next(&page);) { + // Prepare for relocation + page->set_forwarding(); + + // Update pagetable + _pagetable.set_relocating(page); + } +} + +void ZHeap::reset_relocation_set() { + ZRelocationSetIterator iter(&_relocation_set); + for (ZPage* page; iter.next(&page);) { + // Reset relocation information + page->reset_forwarding(); + + // Update pagetable + _pagetable.clear_relocating(page); + } +} + +void ZHeap::relocate_start() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + + // Update statistics + ZStatSample(ZSamplerHeapUsedBeforeRelocation, used()); + + // Flip address view + ZAddressMasks::flip_to_remapped(); + flip_views(); + + // Remap TLABs + _object_allocator.remap_tlabs(); + + // Enter relocate phase + ZGlobalPhase = ZPhaseRelocate; + + // Update statistics + ZStatHeap::set_at_relocate_start(capacity(), allocated(), used()); + + // Remap/Relocate roots + _relocate.start(); +} + +uintptr_t ZHeap::relocate_object(uintptr_t addr) { + assert(ZGlobalPhase == ZPhaseRelocate, "Relocate not allowed"); + ZPage* const page = _pagetable.get(addr); + const bool retained = retain_page(page); + const uintptr_t new_addr = page->relocate_object(addr); + if (retained) { + release_page(page, true /* reclaimed */); + } + + return new_addr; +} + +uintptr_t ZHeap::forward_object(uintptr_t addr) { + assert(ZGlobalPhase == ZPhaseMark || + ZGlobalPhase == ZPhaseMarkCompleted, "Forward not allowed"); + ZPage* const page = _pagetable.get(addr); + return page->forward_object(addr); +} + +void ZHeap::relocate() { + // Relocate relocation set + const bool success = _relocate.relocate(&_relocation_set); + + // Update statistics + ZStatSample(ZSamplerHeapUsedAfterRelocation, used()); + ZStatRelocation::set_at_relocate_end(success); + ZStatHeap::set_at_relocate_end(capacity(), allocated(), reclaimed(), + used(), used_high(), used_low()); +} + +void ZHeap::object_iterate(ObjectClosure* cl) { + // Should only be called in a safepoint after mark end. + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + + ZHeapIterator iter; + iter.objects_do(cl); +} + +void ZHeap::serviceability_initialize() { + _serviceability.initialize(); +} + +GCMemoryManager* ZHeap::serviceability_memory_manager() { + return _serviceability.memory_manager(); +} + +MemoryPool* ZHeap::serviceability_memory_pool() { + return _serviceability.memory_pool(); +} + +ZServiceabilityCounters* ZHeap::serviceability_counters() { + return _serviceability.counters(); +} + +void ZHeap::print_on(outputStream* st) const { + st->print_cr(" ZHeap used " SIZE_FORMAT "M, capacity " SIZE_FORMAT "M, max capacity " SIZE_FORMAT "M", + used() / M, + capacity() / M, + max_capacity() / M); + MetaspaceUtils::print_on(st); +} + +void ZHeap::print_extended_on(outputStream* st) const { + print_on(st); + st->cr(); + + ZPageTableIterator iter(&_pagetable); + for (ZPage* page; iter.next(&page);) { + page->print_on(st); + } + + st->cr(); +} + +class ZVerifyRootsTask : public ZTask { +private: + ZRootsIterator _strong_roots; + ZWeakRootsIterator _weak_roots; + +public: + ZVerifyRootsTask() : + ZTask("ZVerifyRootsTask"), + _strong_roots(), + _weak_roots() {} + + virtual void work() { + ZVerifyRootOopClosure cl; + _strong_roots.oops_do(&cl); + _weak_roots.oops_do(&cl); + } +}; + +void ZHeap::verify() { + // Heap verification can only be done between mark end and + // relocate start. This is the only window where all oop are + // good and the whole heap is in a consistent state. + guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase"); + + { + ZVerifyRootsTask task; + _workers.run_parallel(&task); + } + + { + ZVerifyObjectClosure cl; + object_iterate(&cl); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zHeap.hpp 2018-06-01 22:30:35.524661390 +0200 @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZHEAP_HPP +#define SHARE_GC_Z_ZHEAP_HPP + +#include "gc/shared/gcTimer.hpp" +#include "gc/z/zAllocationFlags.hpp" +#include "gc/z/zArray.hpp" +#include "gc/z/zList.hpp" +#include "gc/z/zLock.hpp" +#include "gc/z/zMark.hpp" +#include "gc/z/zObjectAllocator.hpp" +#include "gc/z/zPage.hpp" +#include "gc/z/zPageAllocator.hpp" +#include "gc/z/zPageTable.hpp" +#include "gc/z/zReferenceProcessor.hpp" +#include "gc/z/zRelocate.hpp" +#include "gc/z/zRelocationSet.hpp" +#include "gc/z/zRelocationSetSelector.hpp" +#include "gc/z/zRootsIterator.hpp" +#include "gc/z/zWeakRootsProcessor.hpp" +#include "gc/z/zServiceability.hpp" +#include "gc/z/zWorkers.hpp" +#include "memory/allocation.hpp" + +class ZHeap { + friend class VMStructs; + +private: + static ZHeap* _heap; + + ZWorkers _workers; + ZObjectAllocator _object_allocator; + ZPageAllocator _page_allocator; + ZPageTable _pagetable; + ZMark _mark; + ZReferenceProcessor _reference_processor; + ZWeakRootsProcessor _weak_roots_processor; + ZRelocate _relocate; + ZRelocationSet _relocation_set; + ZServiceability _serviceability; + + size_t heap_min_size() const; + size_t heap_max_size() const; + size_t heap_max_reserve_size() const; + + void out_of_memory(); + void flip_views(); + void fixup_partial_loads(); + +public: + static ZHeap* heap(); + + ZHeap(); + + bool is_initialized() const; + + // Heap metrics + size_t min_capacity() const; + size_t max_capacity() const; + size_t capacity() const; + size_t max_reserve() const; + size_t used_high() const; + size_t used_low() const; + size_t used() const; + size_t allocated() const; + size_t reclaimed() const; + + size_t tlab_capacity() const; + size_t tlab_used() const; + size_t max_tlab_size() const; + size_t unsafe_max_tlab_alloc() const; + + bool is_in(uintptr_t addr) const; + + // Block + uintptr_t block_start(uintptr_t addr) const; + size_t block_size(uintptr_t addr) const; + bool block_is_obj(uintptr_t addr) const; + + // Workers + uint nconcurrent_worker_threads() const; + uint nconcurrent_no_boost_worker_threads() const; + void set_boost_worker_threads(bool boost); + void worker_threads_do(ThreadClosure* tc) const; + void print_worker_threads_on(outputStream* st) const; + + // Reference processing + ReferenceDiscoverer* reference_discoverer(); + void set_soft_reference_policy(bool clear); + + // Non-strong reference processing + void process_non_strong_references(); + + // Page allocation + ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags); + void undo_alloc_page(ZPage* page); + bool retain_page(ZPage* page); + void release_page(ZPage* page, bool reclaimed); + + // Object allocation + uintptr_t alloc_tlab(size_t size); + uintptr_t alloc_object(size_t size); + uintptr_t alloc_object_for_relocation(size_t size); + void undo_alloc_object_for_relocation(uintptr_t addr, size_t size); + void check_out_of_memory(); + + // Marking + bool is_object_live(uintptr_t addr) const; + bool is_object_strongly_live(uintptr_t addr) const; + template void mark_object(uintptr_t addr); + void mark_start(); + void mark(); + void mark_flush_and_free(Thread* thread); + bool mark_end(); + + // Post-marking & Pre-relocation + void destroy_detached_pages(); + + // Relocation set + void select_relocation_set(); + void prepare_relocation_set(); + void reset_relocation_set(); + + // Relocation + bool is_relocating(uintptr_t addr) const; + void relocate_start(); + uintptr_t relocate_object(uintptr_t addr); + uintptr_t forward_object(uintptr_t addr); + void relocate(); + + // Iteration + void object_iterate(ObjectClosure* cl); + + // Serviceability + void serviceability_initialize(); + GCMemoryManager* serviceability_memory_manager(); + MemoryPool* serviceability_memory_pool(); + ZServiceabilityCounters* serviceability_counters(); + + // Printing + void print_on(outputStream* st) const; + void print_extended_on(outputStream* st) const; + + // Verification + bool is_oop(oop object) const; + void verify(); +}; + +#endif // SHARE_GC_Z_ZHEAP_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zHeap.inline.hpp 2018-06-01 22:30:35.902677707 +0200 @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZHEAP_INLINE_HPP +#define SHARE_GC_Z_ZHEAP_INLINE_HPP + +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zHeap.hpp" +#include "gc/z/zMark.inline.hpp" +#include "gc/z/zOop.inline.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zPageTable.inline.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "utilities/debug.hpp" + +inline ZHeap* ZHeap::heap() { + assert(_heap != NULL, "Not initialized"); + return _heap; +} + +inline ReferenceDiscoverer* ZHeap::reference_discoverer() { + return &_reference_processor; +} + +inline bool ZHeap::is_relocating(uintptr_t addr) const { + return _pagetable.is_relocating(addr); +} + +inline bool ZHeap::is_object_live(uintptr_t addr) const { + ZPage* page = _pagetable.get(addr); + return page->is_object_live(addr); +} + +inline bool ZHeap::is_object_strongly_live(uintptr_t addr) const { + ZPage* page = _pagetable.get(addr); + return page->is_object_strongly_live(addr); +} + +template +inline void ZHeap::mark_object(uintptr_t addr) { + assert(ZGlobalPhase == ZPhaseMark, "Mark not allowed"); + _mark.mark_object(addr); +} + +inline uintptr_t ZHeap::alloc_tlab(size_t size) { + guarantee(size <= max_tlab_size(), "TLAB too large"); + return _object_allocator.alloc_object(size); +} + +inline uintptr_t ZHeap::alloc_object(size_t size) { + uintptr_t addr = _object_allocator.alloc_object(size); + assert(ZAddress::is_good_or_null(addr), "Bad address"); + + if (addr == 0) { + out_of_memory(); + } + + return addr; +} + +inline uintptr_t ZHeap::alloc_object_for_relocation(size_t size) { + uintptr_t addr = _object_allocator.alloc_object_for_relocation(size); + assert(ZAddress::is_good_or_null(addr), "Bad address"); + return addr; +} + +inline void ZHeap::undo_alloc_object_for_relocation(uintptr_t addr, size_t size) { + ZPage* const page = _pagetable.get(addr); + _object_allocator.undo_alloc_object_for_relocation(page, addr, size); +} + +inline void ZHeap::check_out_of_memory() { + _page_allocator.check_out_of_memory(); +} + +inline bool ZHeap::is_oop(oop object) const { + return ZOop::is_good(object); +} + +#endif // SHARE_GC_Z_ZHEAP_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zHeapIterator.cpp 2018-06-01 22:30:36.284694196 +0200 @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zAddressRangeMap.inline.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHeapIterator.hpp" +#include "gc/z/zOop.inline.hpp" +#include "gc/z/zRootsIterator.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/bitMap.inline.hpp" +#include "utilities/stack.inline.hpp" + +class ZHeapIteratorBitMap : public CHeapObj { +private: + CHeapBitMap _map; + +public: + ZHeapIteratorBitMap(size_t size_in_bits) : + _map(size_in_bits) {} + + bool try_set_bit(size_t index) { + if (_map.at(index)) { + return false; + } + + _map.set_bit(index); + return true; + } +}; + +class ZHeapIteratorRootOopClosure : public OopClosure { +private: + ZHeapIterator* const _iter; + ObjectClosure* const _cl; + +public: + ZHeapIteratorRootOopClosure(ZHeapIterator* iter, ObjectClosure* cl) : + _iter(iter), + _cl(cl) {} + + virtual void do_oop(oop* p) { + // Load barrier needed here for the same reason we + // need fixup_partial_loads() in ZHeap::mark_end() + const oop obj = RootAccess<>::oop_load(p); + _iter->push(obj); + _iter->drain(_cl); + } + + virtual void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } +}; + +class ZHeapIteratorPushOopClosure : public ExtendedOopClosure { +private: + ZHeapIterator* const _iter; + const oop _base; + +public: + ZHeapIteratorPushOopClosure(ZHeapIterator* iter, oop base) : + _iter(iter), + _base(base) {} + + void do_oop_nv(oop* p) { + const oop obj = HeapAccess::oop_load_at(_base, _base->field_offset(p)); + _iter->push(obj); + } + + void do_oop_nv(narrowOop* p) { + ShouldNotReachHere(); + } + + virtual void do_oop(oop* p) { + do_oop_nv(p); + } + + virtual void do_oop(narrowOop* p) { + do_oop_nv(p); + } + +#ifdef ASSERT + virtual bool should_verify_oops() { + return false; + } +#endif +}; + +ZHeapIterator::ZHeapIterator() : + _visit_stack(), + _visit_map() {} + +ZHeapIterator::~ZHeapIterator() { + ZVisitMapIterator iter(&_visit_map); + for (ZHeapIteratorBitMap* map; iter.next(&map);) { + delete map; + } +} + +size_t ZHeapIterator::object_index_max() const { + return ZPageSizeMin >> ZObjectAlignmentSmallShift; +} + +size_t ZHeapIterator::object_index(oop obj) const { + const uintptr_t addr = ZOop::to_address(obj); + const uintptr_t offset = ZAddress::offset(addr); + const uintptr_t mask = (1 << ZPageSizeMinShift) - 1; + return (offset & mask) >> ZObjectAlignmentSmallShift; +} + +ZHeapIteratorBitMap* ZHeapIterator::object_map(oop obj) { + const uintptr_t addr = ZOop::to_address(obj); + ZHeapIteratorBitMap* map = _visit_map.get(addr); + if (map == NULL) { + map = new ZHeapIteratorBitMap(object_index_max()); + _visit_map.put(addr, map); + } + + return map; +} + +void ZHeapIterator::push(oop obj) { + if (obj == NULL) { + // Ignore + return; + } + + ZHeapIteratorBitMap* const map = object_map(obj); + const size_t index = object_index(obj); + if (!map->try_set_bit(index)) { + // Already pushed + return; + } + + // Push + _visit_stack.push(obj); +} + +void ZHeapIterator::drain(ObjectClosure* cl) { + while (!_visit_stack.is_empty()) { + const oop obj = _visit_stack.pop(); + + // Visit + cl->do_object(obj); + + // Push members to visit + ZHeapIteratorPushOopClosure push_cl(this, obj); + obj->oop_iterate(&push_cl); + } +} + +void ZHeapIterator::objects_do(ObjectClosure* cl) { + ZHeapIteratorRootOopClosure root_cl(this, cl); + ZRootsIterator roots; + + // Follow roots. Note that we also visit the JVMTI weak tag map + // as if they where strong roots to make sure we visit all tagged + // objects, even those that might now have become unreachable. + // If we didn't do this the user would have expected to see + // ObjectFree events for unreachable objects in the tag map. + roots.oops_do(&root_cl, true /* visit_jvmti_weak_export */); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zHeapIterator.hpp 2018-06-01 22:30:36.673710988 +0200 @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZHEAPITERATOR_HPP +#define SHARE_GC_Z_ZHEAPITERATOR_HPP + +#include "gc/z/zAddressRangeMap.hpp" +#include "gc/z/zGlobals.hpp" +#include "memory/allocation.hpp" +#include "utilities/stack.hpp" + +class ZHeapIteratorBitMap; + +class ZHeapIterator : public StackObj { + friend class ZHeapIteratorRootOopClosure; + friend class ZHeapIteratorPushOopClosure; + +private: + typedef ZAddressRangeMap ZVisitMap; + typedef ZAddressRangeMapIterator ZVisitMapIterator; + typedef Stack ZVisitStack; + + ZVisitStack _visit_stack; + ZVisitMap _visit_map; + + size_t object_index_max() const; + size_t object_index(oop obj) const; + ZHeapIteratorBitMap* object_map(oop obj); + + void push(oop obj); + void drain(ObjectClosure* cl); + +public: + ZHeapIterator(); + ~ZHeapIterator(); + + void objects_do(ObjectClosure* cl); +}; + +#endif // SHARE_GC_Z_ZHEAPITERATOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zInitialize.cpp 2018-06-01 22:30:37.042726916 +0200 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zAddress.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zCPU.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zInitialize.hpp" +#include "gc/z/zLargePages.hpp" +#include "gc/z/zNUMA.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTracer.hpp" +#include "logging/log.hpp" +#include "runtime/vm_version.hpp" + +ZInitialize::ZInitialize(ZBarrierSet* barrier_set) { + log_info(gc, init)("Initializing %s", ZGCName); + log_info(gc, init)("Version: %s (%s)", + Abstract_VM_Version::vm_release(), + Abstract_VM_Version::jdk_debug_level()); + + // Early initialization + ZAddressMasks::initialize(); + ZNUMA::initialize(); + ZCPU::initialize(); + ZStatValue::initialize(); + ZTracer::initialize(); + ZLargePages::initialize(); + ZBarrierSet::set_barrier_set(barrier_set); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zInitialize.hpp 2018-06-01 22:30:37.413742930 +0200 @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZINITIALIZE_HPP +#define SHARE_GC_Z_ZINITIALIZE_HPP + +#include "memory/allocation.hpp" + +class ZBarrierSet; + +class ZInitialize { +public: + ZInitialize(ZBarrierSet* barrier_set); +}; + +#endif // SHARE_GC_Z_ZINITIALIZE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLargePages.cpp 2018-06-01 22:30:37.784758945 +0200 @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zLargePages.hpp" +#include "logging/log.hpp" +#include "runtime/os.hpp" + +ZLargePages::State ZLargePages::_state; + +void ZLargePages::initialize() { + initialize_platform(); + + log_info(gc, init)("Memory: " JULONG_FORMAT "M", os::physical_memory() / M); + log_info(gc, init)("Large Page Support: %s", to_string()); +} + +const char* ZLargePages::to_string() { + switch (_state) { + case Explicit: + return "Enabled (Explicit)"; + + case Transparent: + return "Enabled (Transparent)"; + + default: + return "Disabled"; + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLargePages.hpp 2018-06-01 22:30:38.163775305 +0200 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLARGEPAGES_HPP +#define SHARE_GC_Z_ZLARGEPAGES_HPP + +#include "memory/allocation.hpp" + +class ZLargePages : public AllStatic { +private: + enum State { + Disabled, + Explicit, + Transparent + }; + + static State _state; + + static void initialize_platform(); + +public: + static void initialize(); + + static bool is_enabled(); + static bool is_explicit(); + static bool is_transparent(); + + static const char* to_string(); +}; + +#endif // SHARE_GC_Z_ZLARGEPAGES_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLargePages.inline.hpp 2018-06-01 22:30:38.555792226 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLARGEPAGES_INLINE_HPP +#define SHARE_GC_Z_ZLARGEPAGES_INLINE_HPP + +#include "gc/z/zLargePages.hpp" + +inline bool ZLargePages::is_enabled() { + return _state != Disabled; +} + +inline bool ZLargePages::is_explicit() { + return _state == Explicit; +} + +inline bool ZLargePages::is_transparent() { + return _state == Transparent; +} + +#endif // SHARE_GC_Z_ZLARGEPAGES_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zList.hpp 2018-06-01 22:30:38.937808715 +0200 @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLIST_HPP +#define SHARE_GC_Z_ZLIST_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +template class ZList; + +// Element in a double linked list +template +class ZListNode { + friend class ZList; + +private: + ZListNode* _next; + ZListNode* _prev; + + ZListNode(ZListNode* next, ZListNode* prev) : + _next(next), + _prev(prev) {} + + void set_unused() { + _next = NULL; + _prev = NULL; + } + +public: + ZListNode() { + set_unused(); + } + + ~ZListNode() { + set_unused(); + } + + bool is_unused() const { + return _next == NULL && _prev == NULL; + } +}; + +// Double-linked list +template +class ZList { +private: + ZListNode _head; + size_t _size; + + // Passing by value and assignment is not allowed + ZList(const ZList& list); + ZList& operator=(const ZList& list); + + void verify() const { + assert(_head._next->_prev == &_head, "List corrupt"); + assert(_head._prev->_next == &_head, "List corrupt"); + } + + void insert(ZListNode* before, ZListNode* node) { + verify(); + + assert(node->is_unused(), "Already in a list"); + node->_prev = before; + node->_next = before->_next; + before->_next = node; + node->_next->_prev = node; + + _size++; + } + + ZListNode* cast_to_inner(T* elem) const { + return &elem->_node; + } + + T* cast_to_outer(ZListNode* node) const { + return (T*)((uintptr_t)node - offset_of(T, _node)); + } + +public: + ZList() : + _head(&_head, &_head), + _size(0) { + verify(); + } + + size_t size() const { + verify(); + return _size; + } + + bool is_empty() const { + return _size == 0; + } + + T* first() const { + return is_empty() ? NULL : cast_to_outer(_head._next); + } + + T* last() const { + return is_empty() ? NULL : cast_to_outer(_head._prev); + } + + T* next(T* elem) const { + verify(); + ZListNode* next = cast_to_inner(elem)->_next; + return (next == &_head) ? NULL : cast_to_outer(next); + } + + T* prev(T* elem) const { + verify(); + ZListNode* prev = cast_to_inner(elem)->_prev; + return (prev == &_head) ? NULL : cast_to_outer(prev); + } + + void insert_first(T* elem) { + insert(&_head, cast_to_inner(elem)); + } + + void insert_last(T* elem) { + insert(_head._prev, cast_to_inner(elem)); + } + + void insert_before(T* before, T* elem) { + insert(cast_to_inner(before)->_prev, cast_to_inner(elem)); + } + + void insert_after(T* after, T* elem) { + insert(cast_to_inner(after), cast_to_inner(elem)); + } + + void remove(T* elem) { + verify(); + + ZListNode* const node = cast_to_inner(elem); + assert(!node->is_unused(), "Not in a list"); + + ZListNode* const next = node->_next; + ZListNode* const prev = node->_prev; + assert(next->_prev == node, "List corrupt"); + assert(prev->_next == node, "List corrupt"); + + prev->_next = next; + next->_prev = prev; + node->set_unused(); + + _size--; + } + + T* remove_first() { + T* elem = first(); + if (elem != NULL) { + remove(elem); + } + + return elem; + } + + T* remove_last() { + T* elem = last(); + if (elem != NULL) { + remove(elem); + } + + return elem; + } + + void transfer(ZList* list) { + verify(); + + if (!list->is_empty()) { + list->_head._next->_prev = _head._prev; + list->_head._prev->_next = _head._prev->_next; + + _head._prev->_next = list->_head._next; + _head._prev = list->_head._prev; + + list->_head._next = &list->_head; + list->_head._prev = &list->_head; + + _size += list->_size; + list->_size = 0; + + list->verify(); + verify(); + } + } +}; + +template +class ZListIteratorImpl : public StackObj { +private: + ZList* const _list; + T* _next; + +public: + ZListIteratorImpl(ZList* list); + + bool next(T** elem); +}; + +// Iterator types +#define ZLIST_FORWARD true +#define ZLIST_REVERSE false + +template +class ZListIterator : public ZListIteratorImpl { +public: + ZListIterator(ZList* list) : + ZListIteratorImpl(list) {} +}; + +template +class ZListReverseIterator : public ZListIteratorImpl { +public: + ZListReverseIterator(ZList* list) : + ZListIteratorImpl(list) {} +}; + +#endif // SHARE_GC_Z_ZLIST_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zList.inline.hpp 2018-06-01 22:30:39.316825075 +0200 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLIST_INLINE_HPP +#define SHARE_GC_Z_ZLIST_INLINE_HPP + +#include "gc/z/zList.hpp" + +template +ZListIteratorImpl::ZListIteratorImpl(ZList* list) : + _list(list), + _next(forward ? list->first() : list->last()) {} + +template +bool ZListIteratorImpl::next(T** elem) { + if (_next != NULL) { + *elem = _next; + _next = forward ? _list->next(_next) : _list->prev(_next); + return true; + } + + // No more elements + return false; +} + +#endif // SHARE_GC_Z_ZLIST_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLiveMap.cpp 2018-06-01 22:30:39.700841651 +0200 @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zLiveMap.inline.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zThread.hpp" +#include "logging/log.hpp" +#include "runtime/atomic.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "utilities/debug.hpp" + +static const ZStatCounter ZCounterMarkSeqNumResetContention("Contention", "Mark SeqNum Reset Contention", ZStatUnitOpsPerSecond); +static const ZStatCounter ZCounterMarkSegmentResetContention("Contention", "Mark Segment Reset Contention", ZStatUnitOpsPerSecond); + +ZLiveMap::ZLiveMap(uint32_t size) : + _seqnum(0), + _live_objects(0), + _live_bytes(0), + _segment_live_bits(0), + _segment_claim_bits(0), + // We need at least one bit per segment. + _bitmap(MAX2(size, nsegments) * 2), + _shift(exact_log2(segment_size())) {} + +void ZLiveMap::reset(size_t index) { + const uint32_t seqnum_initializing = (uint32_t)-1; + bool contention = false; + + // Multiple threads can enter here, make sure only one of them + // resets the marking information while the others busy wait. + for (uint32_t seqnum = _seqnum; seqnum != ZGlobalSeqNum; seqnum = _seqnum) { + if ((seqnum != seqnum_initializing) && + (Atomic::cmpxchg(seqnum_initializing, &_seqnum, seqnum) == seqnum)) { + // Reset marking information + _live_bytes = 0; + _live_objects = 0; + + // Clear segment claimed/live bits + segment_live_bits().clear(); + segment_claim_bits().clear(); + + // Make sure the newly reset marking information is + // globally visible before updating the page seqnum. + OrderAccess::storestore(); + + // Update seqnum + assert(_seqnum == seqnum_initializing, "Invalid"); + _seqnum = ZGlobalSeqNum; + break; + } + + // Mark reset contention + if (!contention) { + // Count contention once, not every loop + ZStatInc(ZCounterMarkSeqNumResetContention); + contention = true; + + log_trace(gc)("Mark seqnum reset contention, thread: " PTR_FORMAT " (%s), map: " PTR_FORMAT ", bit: " SIZE_FORMAT, + ZThread::id(), ZThread::name(), p2i(this), index); + } + } +} + +void ZLiveMap::reset_segment(BitMap::idx_t segment) { + bool contention = false; + + if (!claim_segment(segment)) { + // Already claimed, wait for live bit to be set + while (!is_segment_live(segment)) { + // Busy wait. The loadload barrier is needed to make + // sure we re-read the live bit every time we loop. + OrderAccess::loadload(); + + // Mark reset contention + if (!contention) { + // Count contention once, not every loop + ZStatInc(ZCounterMarkSegmentResetContention); + contention = true; + + log_trace(gc)("Mark segment reset contention, thread: " PTR_FORMAT " (%s), map: " PTR_FORMAT ", segment: " SIZE_FORMAT, + ZThread::id(), ZThread::name(), p2i(this), segment); + } + } + + // Segment is live + return; + } + + // Segment claimed, clear it + const BitMap::idx_t start_index = segment_start(segment); + const BitMap::idx_t end_index = segment_end(segment); + if (segment_size() / BitsPerWord >= 32) { + _bitmap.clear_large_range(start_index, end_index); + } else { + _bitmap.clear_range(start_index, end_index); + } + + // Set live bit + const bool success = set_segment_live_atomic(segment); + assert(success, "Should never fail"); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLiveMap.hpp 2018-06-01 22:30:40.079858011 +0200 @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLIVEMAP_HPP +#define SHARE_GC_Z_ZLIVEMAP_HPP + +#include "gc/z/zBitMap.hpp" +#include "memory/allocation.hpp" + +class ObjectClosure; + +class ZLiveMap { + friend class ZLiveMapTest; + +private: + static const size_t nsegments = 64; + + volatile uint32_t _seqnum; // Mark sequence number + volatile uint32_t _live_objects; // Number of live objects + volatile size_t _live_bytes; // Number of live bytes + BitMap::bm_word_t _segment_live_bits; // Segment live bits + BitMap::bm_word_t _segment_claim_bits; // Segment claim bits + ZBitMap _bitmap; // Mark bitmap + const size_t _shift; // Segment shift + + const BitMapView segment_live_bits() const; + const BitMapView segment_claim_bits() const; + + BitMapView segment_live_bits(); + BitMapView segment_claim_bits(); + + BitMap::idx_t segment_size() const; + + BitMap::idx_t segment_start(BitMap::idx_t segment) const; + BitMap::idx_t segment_end(BitMap::idx_t segment) const; + + bool is_segment_live(BitMap::idx_t segment) const; + bool set_segment_live_atomic(BitMap::idx_t segment); + + BitMap::idx_t first_live_segment() const; + BitMap::idx_t next_live_segment(BitMap::idx_t segment) const; + BitMap::idx_t index_to_segment(BitMap::idx_t index) const; + + bool claim_segment(BitMap::idx_t segment); + + void reset(size_t index); + void reset_segment(BitMap::idx_t segment); + + void iterate_segment(ObjectClosure* cl, BitMap::idx_t segment, uintptr_t page_start, size_t page_object_alignment_shift); + +public: + ZLiveMap(uint32_t size); + + void reset(); + + bool is_marked() const; + + uint32_t live_objects() const; + size_t live_bytes() const; + + bool get(size_t index) const; + bool set_atomic(size_t index, bool finalizable, bool& inc_live); + + void inc_live_atomic(uint32_t objects, size_t bytes); + + void iterate(ObjectClosure* cl, uintptr_t page_start, size_t page_object_alignment_shift); +}; + +#endif // SHARE_GC_Z_ZLIVEMAP_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLiveMap.inline.hpp 2018-06-01 22:30:40.463874587 +0200 @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLIVEMAP_INLINE_HPP +#define SHARE_GC_Z_ZLIVEMAP_INLINE_HPP + +#include "gc/z/zBitMap.inline.hpp" +#include "gc/z/zLiveMap.hpp" +#include "gc/z/zMark.hpp" +#include "gc/z/zOop.inline.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "runtime/atomic.hpp" +#include "utilities/bitMap.inline.hpp" +#include "utilities/debug.hpp" + +inline void ZLiveMap::reset() { + _seqnum = 0; +} + +inline bool ZLiveMap::is_marked() const { + return _seqnum == ZGlobalSeqNum; +} + +inline uint32_t ZLiveMap::live_objects() const { + assert(ZGlobalPhase != ZPhaseMark, "Invalid phase"); + return _live_objects; +} + +inline size_t ZLiveMap::live_bytes() const { + assert(ZGlobalPhase != ZPhaseMark, "Invalid phase"); + return _live_bytes; +} + +inline const BitMapView ZLiveMap::segment_live_bits() const { + return BitMapView(const_cast(&_segment_live_bits), nsegments); +} + +inline const BitMapView ZLiveMap::segment_claim_bits() const { + return BitMapView(const_cast(&_segment_claim_bits), nsegments); +} + +inline BitMapView ZLiveMap::segment_live_bits() { + return BitMapView(&_segment_live_bits, nsegments); +} + +inline BitMapView ZLiveMap::segment_claim_bits() { + return BitMapView(&_segment_claim_bits, nsegments); +} + +inline bool ZLiveMap::is_segment_live(BitMap::idx_t segment) const { + return segment_live_bits().at(segment); +} + +inline bool ZLiveMap::set_segment_live_atomic(BitMap::idx_t segment) { + return segment_live_bits().par_set_bit(segment); +} + +inline bool ZLiveMap::claim_segment(BitMap::idx_t segment) { + return segment_claim_bits().par_set_bit(segment); +} + +inline BitMap::idx_t ZLiveMap::first_live_segment() const { + return segment_live_bits().get_next_one_offset(0, nsegments); +} + +inline BitMap::idx_t ZLiveMap::next_live_segment(BitMap::idx_t segment) const { + return segment_live_bits().get_next_one_offset(segment + 1, nsegments); +} + +inline BitMap::idx_t ZLiveMap::segment_size() const { + return _bitmap.size() / nsegments; +} + +inline BitMap::idx_t ZLiveMap::index_to_segment(BitMap::idx_t index) const { + return index >> _shift; +} + +inline bool ZLiveMap::get(size_t index) const { + BitMap::idx_t segment = index_to_segment(index); + return is_marked() && // Page is marked + is_segment_live(segment) && // Segment is marked + _bitmap.at(index); // Object is marked +} + +inline bool ZLiveMap::set_atomic(size_t index, bool finalizable, bool& inc_live) { + if (!is_marked()) { + // First object to be marked during this + // cycle, reset marking information. + reset(index); + } + + const BitMap::idx_t segment = index_to_segment(index); + if (!is_segment_live(segment)) { + // First object to be marked in this segment during + // this cycle, reset segment bitmap. + reset_segment(segment); + } + + return _bitmap.par_set_bit_pair(index, finalizable, inc_live); +} + +inline void ZLiveMap::inc_live_atomic(uint32_t objects, size_t bytes) { + Atomic::add(objects, &_live_objects); + Atomic::add(bytes, &_live_bytes); +} + +inline BitMap::idx_t ZLiveMap::segment_start(BitMap::idx_t segment) const { + return segment_size() * segment; +} + +inline BitMap::idx_t ZLiveMap::segment_end(BitMap::idx_t segment) const { + return segment_start(segment) + segment_size(); +} + +inline void ZLiveMap::iterate_segment(ObjectClosure* cl, BitMap::idx_t segment, uintptr_t page_start, size_t page_object_alignment_shift) { + assert(is_segment_live(segment), "Must be"); + + const BitMap::idx_t start_index = segment_start(segment); + const BitMap::idx_t end_index = segment_end(segment); + BitMap::idx_t index = _bitmap.get_next_one_offset(start_index, end_index); + + while (index < end_index) { + // Calculate object address + const uintptr_t addr = page_start + ((index / 2) << page_object_alignment_shift); + + // Apply closure + cl->do_object(ZOop::to_oop(addr)); + + // Find next bit after this object + const size_t size = ZUtils::object_size(addr); + const uintptr_t next_addr = align_up(addr + size, 1 << page_object_alignment_shift); + const BitMap::idx_t next_index = ((next_addr - page_start) >> page_object_alignment_shift) * 2; + if (next_index >= end_index) { + // End of live map + break; + } + + index = _bitmap.get_next_one_offset(next_index, end_index); + } +} + +inline void ZLiveMap::iterate(ObjectClosure* cl, uintptr_t page_start, size_t page_object_alignment_shift) { + if (is_marked()) { + for (BitMap::idx_t segment = first_live_segment(); segment < nsegments; segment = next_live_segment(segment)) { + // For each live segment + iterate_segment(cl, segment, page_start, page_object_alignment_shift); + } + } +} + +#endif // SHARE_GC_Z_ZLIVEMAP_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLock.hpp 2018-06-01 22:30:40.853891421 +0200 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLOCK_HPP +#define SHARE_GC_Z_ZLOCK_HPP + +#include "memory/allocation.hpp" +#include + +class ZLock { +private: + pthread_mutex_t _lock; + +public: + ZLock(); + + void lock(); + bool try_lock(); + void unlock(); +}; + +class ZLocker : public StackObj { +private: + ZLock* const _lock; + +public: + ZLocker(ZLock* lock); + ~ZLocker(); +}; + +#endif // SHARE_GC_Z_ZLOCK_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zLock.inline.hpp 2018-06-01 22:30:41.227907566 +0200 @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZLOCK_INLINE_HPP +#define SHARE_GC_Z_ZLOCK_INLINE_HPP + +#include "gc/z/zLock.hpp" + +inline ZLock::ZLock() { + pthread_mutex_init(&_lock, NULL); +} + +inline void ZLock::lock() { + pthread_mutex_lock(&_lock); +} + +inline bool ZLock::try_lock() { + return pthread_mutex_trylock(&_lock) == 0; +} + +inline void ZLock::unlock() { + pthread_mutex_unlock(&_lock); +} + +inline ZLocker::ZLocker(ZLock* lock) : + _lock(lock) { + _lock->lock(); +} + +inline ZLocker::~ZLocker() { + _lock->unlock(); +} + +#endif // SHARE_GC_Z_ZLOCK_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMark.cpp 2018-06-01 22:30:41.598923580 +0200 @@ -0,0 +1,684 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zMark.inline.hpp" +#include "gc/z/zMarkCache.inline.hpp" +#include "gc/z/zMarkStack.inline.hpp" +#include "gc/z/zMarkTerminate.inline.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zPage.hpp" +#include "gc/z/zPageTable.inline.hpp" +#include "gc/z/zRootsIterator.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zThread.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "gc/z/zWorkers.inline.hpp" +#include "logging/log.hpp" +#include "oops/objArrayOop.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.hpp" +#include "runtime/handshake.hpp" +#include "runtime/orderAccess.hpp" +#include "runtime/prefetch.inline.hpp" +#include "runtime/thread.hpp" +#include "utilities/align.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ticks.hpp" + +static const ZStatSubPhase ZSubPhaseConcurrentMark("Concurrent Mark"); +static const ZStatSubPhase ZSubPhaseConcurrentMarkTryFlush("Concurrent Mark Try Flush"); +static const ZStatSubPhase ZSubPhaseConcurrentMarkIdle("Concurrent Mark Idle"); +static const ZStatSubPhase ZSubPhaseConcurrentMarkTryTerminate("Concurrent Mark Try Terminate"); +static const ZStatSubPhase ZSubPhaseMarkTryComplete("Pause Mark Try Complete"); + +ZMark::ZMark(ZWorkers* workers, ZPageTable* pagetable) : + _workers(workers), + _pagetable(pagetable), + _allocator(), + _stripes(), + _terminate(), + _work_terminateflush(true), + _work_nproactiveflush(0), + _work_nterminateflush(0), + _nproactiveflush(0), + _nterminateflush(0), + _ntrycomplete(0), + _ncontinue(0), + _nworkers(0) {} + +size_t ZMark::calculate_nstripes(uint nworkers) const { + // Calculate the number of stripes from the number of workers we use, + // where the number of stripes must be a power of two and we want to + // have at least one worker per stripe. + const size_t nstripes = ZUtils::round_down_power_of_2(nworkers); + return MIN2(nstripes, ZMarkStripesMax); +} + +void ZMark::prepare_mark() { + // Increment global sequence number to invalidate + // marking information for all pages. + ZGlobalSeqNum++; + + // Reset flush/continue counters + _nproactiveflush = 0; + _nterminateflush = 0; + _ntrycomplete = 0; + _ncontinue = 0; + + // Set number of workers to use + _nworkers = _workers->nconcurrent(); + + // Set number of mark stripes to use, based on number + // of workers we will use in the concurrent mark phase. + const size_t nstripes = calculate_nstripes(_nworkers); + _stripes.set_nstripes(nstripes); + + // Update statistics + ZStatMark::set_at_mark_start(nstripes); + + // Print worker/stripe distribution + LogTarget(Debug, gc, marking) log; + if (log.is_enabled()) { + log.print("Mark Worker/Stripe Distribution"); + for (uint worker_id = 0; worker_id < _nworkers; worker_id++) { + const ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, worker_id); + const size_t stripe_id = _stripes.stripe_id(stripe); + log.print(" Worker %u(%u) -> Stripe " SIZE_FORMAT "(" SIZE_FORMAT ")", + worker_id, _nworkers, stripe_id, nstripes); + } + } +} + +class ZMarkRootsTask : public ZTask { +private: + ZMark* const _mark; + ZRootsIterator _roots; + +public: + ZMarkRootsTask(ZMark* mark) : + ZTask("ZMarkRootsTask"), + _mark(mark), + _roots() {} + + virtual void work() { + ZMarkRootOopClosure cl; + _roots.oops_do(&cl); + + // Flush and free worker stacks. Needed here since + // the set of workers executing during root scanning + // can be different from the set of workers executing + // during mark. + _mark->flush_and_free(); + } +}; + +void ZMark::start() { + // Verification + if (ZVerifyMarking) { + verify_all_stacks_empty(); + } + + // Prepare for concurrent mark + prepare_mark(); + + // Mark roots + ZMarkRootsTask task(this); + _workers->run_parallel(&task); +} + +void ZMark::prepare_work() { + assert(_nworkers == _workers->nconcurrent(), "Invalid number of workers"); + + // Set number of active workers + _terminate.reset(_nworkers); + + // Reset flush counters + _work_nproactiveflush = _work_nterminateflush = 0; + _work_terminateflush = true; +} + +void ZMark::finish_work() { + // Accumulate proactive/terminate flush counters + _nproactiveflush += _work_nproactiveflush; + _nterminateflush += _work_nterminateflush; +} + +bool ZMark::is_array(uintptr_t addr) const { + return ZOop::to_oop(addr)->is_objArray(); +} + +void ZMark::push_partial_array(uintptr_t addr, size_t size, bool finalizable) { + assert(is_aligned(addr, ZMarkPartialArrayMinSize), "Address misaligned"); + ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current()); + ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr); + const uintptr_t offset = ZAddress::offset(addr) >> ZMarkPartialArrayMinSizeShift; + const uintptr_t length = size / oopSize; + const ZMarkStackEntry entry(offset, length, finalizable); + + log_develop_trace(gc, marking)("Array push partial: " PTR_FORMAT " (" SIZE_FORMAT "), stripe: " SIZE_FORMAT, + addr, size, _stripes.stripe_id(stripe)); + + stacks->push(&_allocator, &_stripes, stripe, entry, false /* publish */); +} + +void ZMark::follow_small_array(uintptr_t addr, size_t size, bool finalizable) { + assert(size <= ZMarkPartialArrayMinSize, "Too large, should be split"); + const size_t length = size / oopSize; + + log_develop_trace(gc, marking)("Array follow small: " PTR_FORMAT " (" SIZE_FORMAT ")", addr, size); + + ZBarrier::mark_barrier_on_oop_array((oop*)addr, length, finalizable); +} + +void ZMark::follow_large_array(uintptr_t addr, size_t size, bool finalizable) { + assert(size <= (size_t)arrayOopDesc::max_array_length(T_OBJECT) * oopSize, "Too large"); + assert(size > ZMarkPartialArrayMinSize, "Too small, should not be split"); + const uintptr_t start = addr; + const uintptr_t end = start + size; + + // Calculate the aligned middle start/end/size, where the middle start + // should always be greater than the start (hence the +1 below) to make + // sure we always do some follow work, not just split the array into pieces. + const uintptr_t middle_start = align_up(start + 1, ZMarkPartialArrayMinSize); + const size_t middle_size = align_down(end - middle_start, ZMarkPartialArrayMinSize); + const uintptr_t middle_end = middle_start + middle_size; + + log_develop_trace(gc, marking)("Array follow large: " PTR_FORMAT "-" PTR_FORMAT" (" SIZE_FORMAT "), " + "middle: " PTR_FORMAT "-" PTR_FORMAT " (" SIZE_FORMAT ")", + start, end, size, middle_start, middle_end, middle_size); + + // Push unaligned trailing part + if (end > middle_end) { + const uintptr_t trailing_addr = middle_end; + const size_t trailing_size = end - middle_end; + push_partial_array(trailing_addr, trailing_size, finalizable); + } + + // Push aligned middle part(s) + uintptr_t partial_addr = middle_end; + while (partial_addr > middle_start) { + const size_t parts = 2; + const size_t partial_size = align_up((partial_addr - middle_start) / parts, ZMarkPartialArrayMinSize); + partial_addr -= partial_size; + push_partial_array(partial_addr, partial_size, finalizable); + } + + // Follow leading part + assert(start < middle_start, "Miscalculated middle start"); + const uintptr_t leading_addr = start; + const size_t leading_size = middle_start - start; + follow_small_array(leading_addr, leading_size, finalizable); +} + +void ZMark::follow_array(uintptr_t addr, size_t size, bool finalizable) { + if (size <= ZMarkPartialArrayMinSize) { + follow_small_array(addr, size, finalizable); + } else { + follow_large_array(addr, size, finalizable); + } +} + +void ZMark::follow_partial_array(ZMarkStackEntry entry, bool finalizable) { + const uintptr_t addr = ZAddress::good(entry.partial_array_offset() << ZMarkPartialArrayMinSizeShift); + const size_t size = entry.partial_array_length() * oopSize; + + follow_array(addr, size, finalizable); +} + +void ZMark::follow_array_object(objArrayOop obj, bool finalizable) { + const uintptr_t addr = (uintptr_t)obj->base(); + const size_t size = (size_t)obj->length() * oopSize; + + follow_array(addr, size, finalizable); +} + +void ZMark::follow_object(oop obj, bool finalizable) { + if (finalizable) { + ZMarkBarrierOopClosure cl; + obj->oop_iterate(&cl); + } else { + ZMarkBarrierOopClosure cl; + obj->oop_iterate(&cl); + } +} + +bool ZMark::try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable) { + ZPage* const page = _pagetable->get(addr); + if (page->is_allocating()) { + // Newly allocated objects are implicitly marked + return false; + } + + // Try mark object + bool inc_live = false; + const bool success = page->mark_object(addr, finalizable, inc_live); + if (inc_live) { + // Update live objects/bytes for page. We use the aligned object + // size since that is the actual number of bytes used on the page + // and alignment paddings can never be reclaimed. + const size_t size = ZUtils::object_size(addr); + const size_t aligned_size = align_up(size, page->object_alignment()); + cache->inc_live(page, aligned_size); + } + + return success; +} + +void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) { + // Decode flags + const bool finalizable = entry.finalizable(); + const bool partial_array = entry.partial_array(); + + if (partial_array) { + follow_partial_array(entry, finalizable); + return; + } + + // Decode object address + const uintptr_t addr = entry.object_address(); + + if (!try_mark_object(cache, addr, finalizable)) { + // Already marked + return; + } + + if (is_array(addr)) { + follow_array_object(objArrayOop(ZOop::to_oop(addr)), finalizable); + } else { + follow_object(ZOop::to_oop(addr), finalizable); + } +} + +template +bool ZMark::drain(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, ZMarkCache* cache, T* timeout) { + ZMarkStackEntry entry; + + // Drain stripe stacks + while (stacks->pop(&_allocator, &_stripes, stripe, entry)) { + mark_and_follow(cache, entry); + + // Check timeout + if (timeout->has_expired()) { + // Timeout + return false; + } + } + + // Success + return true; +} + +template +bool ZMark::drain_and_flush(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, ZMarkCache* cache, T* timeout) { + const bool success = drain(stripe, stacks, cache, timeout); + + // Flush and publish worker stacks + stacks->flush(&_allocator, &_stripes); + + return success; +} + +bool ZMark::try_steal(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks) { + // Try to steal a stack from another stripe + for (ZMarkStripe* victim_stripe = _stripes.stripe_next(stripe); + victim_stripe != stripe; + victim_stripe = _stripes.stripe_next(victim_stripe)) { + ZMarkStack* const stack = victim_stripe->steal_stack(); + if (stack != NULL) { + // Success, install the stolen stack + stacks->install(&_stripes, stripe, stack); + return true; + } + } + + // Nothing to steal + return false; +} + +void ZMark::idle() const { + ZStatTimer timer(ZSubPhaseConcurrentMarkIdle); + os::naked_short_sleep(1); +} + +class ZMarkFlushAndFreeStacksClosure : public ThreadClosure { +private: + ZMark* const _mark; + bool _flushed; + +public: + ZMarkFlushAndFreeStacksClosure(ZMark* mark) : + _mark(mark), + _flushed(false) {} + + void do_thread(Thread* thread) { + if (_mark->flush_and_free(thread)) { + _flushed = true; + } + } + + bool flushed() const { + return _flushed; + } +}; + +bool ZMark::flush(bool at_safepoint) { + ZMarkFlushAndFreeStacksClosure cl(this); + if (at_safepoint) { + Threads::threads_do(&cl); + } else { + Handshake::execute(&cl); + } + + // Returns true if more work is available + return cl.flushed() || !_stripes.is_empty(); +} + +bool ZMark::try_flush(volatile size_t* nflush) { + // Only flush if handhakes are enabled + if (!ThreadLocalHandshakes) { + return false; + } + + Atomic::inc(nflush); + + ZStatTimer timer(ZSubPhaseConcurrentMarkTryFlush); + return flush(false /* at_safepoint */); +} + +bool ZMark::try_proactive_flush() { + // Only do proactive flushes from worker 0 + if (ZThread::worker_id() != 0) { + return false; + } + + if (Atomic::load(&_work_nproactiveflush) == ZMarkProactiveFlushMax || + Atomic::load(&_work_nterminateflush) != 0) { + // Limit reached or we're trying to terminate + return false; + } + + return try_flush(&_work_nproactiveflush); +} + +bool ZMark::try_terminate() { + ZStatTimer timer(ZSubPhaseConcurrentMarkTryTerminate); + + if (_terminate.enter_stage0()) { + // Last thread entered stage 0, flush + if (Atomic::load(&_work_terminateflush) && + Atomic::load(&_work_nterminateflush) != ZMarkTerminateFlushMax) { + // Exit stage 0 to allow other threads to continue marking + _terminate.exit_stage0(); + + // Flush before termination + if (!try_flush(&_work_nterminateflush)) { + // No more work available, skip further flush attempts + Atomic::store(false, &_work_terminateflush); + } + + // Don't terminate, regardless of whether we successfully + // flushed out more work or not. We've already exited + // termination stage 0, to allow other threads to continue + // marking, so this thread has to return false and also + // make another round of attempted marking. + return false; + } + } + + for (;;) { + if (_terminate.enter_stage1()) { + // Last thread entered stage 1, terminate + return true; + } + + // Idle to give the other threads + // a chance to enter termination. + idle(); + + if (!_terminate.try_exit_stage1()) { + // All workers in stage 1, terminate + return true; + } + + if (_terminate.try_exit_stage0()) { + // More work available, don't terminate + return false; + } + } +} + +class ZMarkNoTimeout : public StackObj { +public: + bool has_expired() { + return false; + } +}; + +void ZMark::work_without_timeout(ZMarkCache* cache, ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks) { + ZStatTimer timer(ZSubPhaseConcurrentMark); + ZMarkNoTimeout no_timeout; + + for (;;) { + drain_and_flush(stripe, stacks, cache, &no_timeout); + + if (try_steal(stripe, stacks)) { + // Stole work + continue; + } + + if (try_proactive_flush()) { + // Work available + continue; + } + + if (try_terminate()) { + // Terminate + break; + } + } +} + +class ZMarkTimeout : public StackObj { +private: + const Ticks _start; + const uint64_t _timeout; + const uint64_t _check_interval; + uint64_t _check_at; + uint64_t _check_count; + bool _expired; + +public: + ZMarkTimeout(uint64_t timeout_in_millis) : + _start(Ticks::now()), + _timeout(_start.value() + TimeHelper::millis_to_counter(timeout_in_millis)), + _check_interval(200), + _check_at(_check_interval), + _check_count(0), + _expired(false) {} + + ~ZMarkTimeout() { + const Tickspan duration = Ticks::now() - _start; + log_debug(gc, marking)("Mark With Timeout (%s): %s, " UINT64_FORMAT " oops, %.3fms", + ZThread::name(), _expired ? "Expired" : "Completed", + _check_count, TimeHelper::counter_to_millis(duration.value())); + } + + bool has_expired() { + if (++_check_count == _check_at) { + _check_at += _check_interval; + if ((uint64_t)Ticks::now().value() >= _timeout) { + // Timeout + _expired = true; + } + } + + return _expired; + } +}; + +void ZMark::work_with_timeout(ZMarkCache* cache, ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, uint64_t timeout_in_millis) { + ZStatTimer timer(ZSubPhaseMarkTryComplete); + ZMarkTimeout timeout(timeout_in_millis); + + for (;;) { + if (!drain_and_flush(stripe, stacks, cache, &timeout)) { + // Timed out + break; + } + + if (try_steal(stripe, stacks)) { + // Stole work + continue; + } + + // Terminate + break; + } +} + +void ZMark::work(uint64_t timeout_in_millis) { + ZMarkCache cache(_stripes.nstripes()); + ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, ZThread::worker_id()); + ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current()); + + if (timeout_in_millis == 0) { + work_without_timeout(&cache, stripe, stacks); + } else { + work_with_timeout(&cache, stripe, stacks, timeout_in_millis); + } + + // Make sure stacks have been flushed + assert(stacks->is_empty(&_stripes), "Should be empty"); + + // Free remaining stacks + stacks->free(&_allocator); +} + +class ZMarkTask : public ZTask { +private: + ZMark* const _mark; + const uint64_t _timeout_in_millis; + +public: + ZMarkTask(ZMark* mark, uint64_t timeout_in_millis = 0) : + ZTask("ZMarkTask"), + _mark(mark), + _timeout_in_millis(timeout_in_millis) { + _mark->prepare_work(); + } + + ~ZMarkTask() { + _mark->finish_work(); + } + + virtual void work() { + _mark->work(_timeout_in_millis); + } +}; + +void ZMark::mark() { + ZMarkTask task(this); + _workers->run_concurrent(&task); +} + +bool ZMark::try_complete() { + _ntrycomplete++; + + // Use nconcurrent number of worker threads to maintain the + // worker/stripe distribution used during concurrent mark. + ZMarkTask task(this, ZMarkCompleteTimeout); + _workers->run_concurrent(&task); + + // Successful if all stripes are empty + return _stripes.is_empty(); +} + +bool ZMark::try_end() { + // Flush all mark stacks + if (!flush(true /* at_safepoint */)) { + // Mark completed + return true; + } + + // Try complete marking by doing a limited + // amount of mark work in this phase. + return try_complete(); +} + +bool ZMark::end() { + // Try end marking + if (!try_end()) { + // Mark not completed + _ncontinue++; + return false; + } + + // Verification + if (ZVerifyMarking) { + verify_all_stacks_empty(); + } + + // Update statistics + ZStatMark::set_at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue); + + // Mark completed + return true; +} + +void ZMark::flush_and_free() { + Thread* const thread = Thread::current(); + flush_and_free(thread); +} + +bool ZMark::flush_and_free(Thread* thread) { + ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread); + const bool flushed = stacks->flush(&_allocator, &_stripes); + stacks->free(&_allocator); + return flushed; +} + +class ZVerifyMarkStacksEmptyClosure : public ThreadClosure { +private: + const ZMarkStripeSet* const _stripes; + +public: + ZVerifyMarkStacksEmptyClosure(const ZMarkStripeSet* stripes) : + _stripes(stripes) {} + + void do_thread(Thread* thread) { + ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread); + guarantee(stacks->is_empty(_stripes), "Should be empty"); + } +}; + +void ZMark::verify_all_stacks_empty() const { + // Verify thread stacks + ZVerifyMarkStacksEmptyClosure cl(&_stripes); + Threads::threads_do(&cl); + + // Verify stripe stacks + guarantee(_stripes.is_empty(), "Should be emtpy"); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMark.hpp 2018-06-01 22:30:41.982940156 +0200 @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARK_HPP +#define SHARE_GC_Z_ZMARK_HPP + +#include "gc/z/zMarkStack.hpp" +#include "gc/z/zMarkTerminate.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/globalDefinitions.hpp" + +class Thread; +class ZMarkCache; +class ZPageTable; +class ZWorkers; + +class ZMark { + friend class ZMarkRootsTask; + friend class ZMarkTask; + friend class ZMarkTryCompleteTask; + +private: + ZWorkers* const _workers; + ZPageTable* const _pagetable; + ZMarkStackAllocator _allocator; + ZMarkStripeSet _stripes; + ZMarkTerminate _terminate; + volatile bool _work_terminateflush; + volatile size_t _work_nproactiveflush; + volatile size_t _work_nterminateflush; + size_t _nproactiveflush; + size_t _nterminateflush; + size_t _ntrycomplete; + size_t _ncontinue; + uint _nworkers; + + size_t calculate_nstripes(uint nworkers) const; + void prepare_mark(); + + bool is_array(uintptr_t addr) const; + void push_partial_array(uintptr_t addr, size_t size, bool finalizable); + void follow_small_array(uintptr_t addr, size_t size, bool finalizable); + void follow_large_array(uintptr_t addr, size_t size, bool finalizable); + void follow_array(uintptr_t addr, size_t size, bool finalizable); + void follow_partial_array(ZMarkStackEntry entry, bool finalizable); + void follow_array_object(objArrayOop obj, bool finalizable); + void follow_object(oop obj, bool finalizable); + bool try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable); + void mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry); + + template bool drain(ZMarkStripe* stripe, + ZMarkThreadLocalStacks* stacks, + ZMarkCache* cache, + T* timeout); + template bool drain_and_flush(ZMarkStripe* stripe, + ZMarkThreadLocalStacks* stacks, + ZMarkCache* cache, + T* timeout); + bool try_steal(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks); + void idle() const; + bool flush(bool at_safepoint); + bool try_proactive_flush(); + bool try_flush(volatile size_t* nflush); + bool try_terminate(); + bool try_complete(); + bool try_end(); + + void prepare_work(); + void finish_work(); + + void work_without_timeout(ZMarkCache* cache, + ZMarkStripe* stripe, + ZMarkThreadLocalStacks* stacks); + void work_with_timeout(ZMarkCache* cache, + ZMarkStripe* stripe, + ZMarkThreadLocalStacks* stacks, + uint64_t timeout_in_millis); + void work(uint64_t timeout_in_millis); + + void verify_all_stacks_empty() const; + +public: + ZMark(ZWorkers* workers, ZPageTable* pagetable); + + template void mark_object(uintptr_t addr); + + void start(); + void mark(); + bool end(); + + void flush_and_free(); + bool flush_and_free(Thread* thread); +}; + +#endif // SHARE_GC_Z_ZMARK_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMark.inline.hpp 2018-06-01 22:30:42.373957034 +0200 @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARK_INLINE_HPP +#define SHARE_GC_Z_ZMARK_INLINE_HPP + +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zMark.hpp" +#include "gc/z/zMarkStack.inline.hpp" +#include "gc/z/zThreadLocalData.hpp" +#include "runtime/thread.hpp" +#include "utilities/debug.hpp" + +template +inline void ZMark::mark_object(uintptr_t addr) { + assert(ZAddress::is_marked(addr), "Should be marked"); + ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current()); + ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr); + ZMarkStackEntry entry(addr, finalizable); + + stacks->push(&_allocator, &_stripes, stripe, entry, publish); +} + +#endif // SHARE_GC_Z_ZMARK_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkCache.cpp 2018-06-01 22:30:42.762973825 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zMarkCache.inline.hpp" +#include "utilities/globalDefinitions.hpp" + +ZMarkCacheEntry::ZMarkCacheEntry() : + _page(NULL), + _objects(0), + _bytes(0) {} + +ZMarkCache::ZMarkCache(size_t nstripes) : + _shift(ZMarkStripeShift + exact_log2(nstripes)) {} + +ZMarkCache::~ZMarkCache() { + // Evict all entries + for (size_t i = 0; i < ZMarkCacheSize; i++) { + _cache[i].evict(); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkCache.hpp 2018-06-01 22:30:43.140990142 +0200 @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKCACHE_HPP +#define SHARE_GC_Z_ZMARKCACHE_HPP + +#include "gc/z/zGlobals.hpp" +#include "memory/allocation.hpp" + +class ZPage; + +class ZMarkCacheEntry { +private: + ZPage* _page; + uint32_t _objects; + size_t _bytes; + +public: + ZMarkCacheEntry(); + + void inc_live(ZPage* page, size_t bytes); + void evict(); +}; + +class ZMarkCache : public StackObj { +private: + const size_t _shift; + ZMarkCacheEntry _cache[ZMarkCacheSize]; + +public: + ZMarkCache(size_t nstripes); + ~ZMarkCache(); + + void inc_live(ZPage* page, size_t bytes); +}; + +#endif // SHARE_GC_Z_ZMARKCACHE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkCache.inline.hpp 2018-06-01 22:30:43.529006891 +0200 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKCACHE_INLINE_HPP +#define SHARE_GC_Z_ZMARKCACHE_INLINE_HPP + +#include "gc/z/zMarkCache.hpp" +#include "gc/z/zPage.inline.hpp" + +inline void ZMarkCacheEntry::inc_live(ZPage* page, size_t bytes) { + if (_page == page) { + // Cache hit + _objects++; + _bytes += bytes; + } else { + // Cache miss + evict(); + _page = page; + _objects = 1; + _bytes = bytes; + } +} + +inline void ZMarkCacheEntry::evict() { + if (_page != NULL) { + // Write cached data out to page + _page->inc_live_atomic(_objects, _bytes); + _page = NULL; + } +} + +inline void ZMarkCache::inc_live(ZPage* page, size_t bytes) { + const size_t mask = ZMarkCacheSize - 1; + const size_t index = (page->start() >> _shift) & mask; + _cache[index].inc_live(page, bytes); +} + +#endif // SHARE_GC_Z_ZMARKCACHE_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkStack.cpp 2018-06-01 22:30:43.928024114 +0200 @@ -0,0 +1,393 @@ +/* + * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zErrno.hpp" +#include "gc/z/zFlags.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zLock.inline.hpp" +#include "gc/z/zMarkStack.inline.hpp" +#include "logging/log.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" + +#include +#include + +ZMarkStackSpace::ZMarkStackSpace() : + _expand_lock(), + _top(0), + _end(0) { + assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small"); + assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large"); + + // Reserve address space + const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize, + PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); + if (res != (void*)ZMarkStackSpaceStart) { + log_error(gc, marking)("Failed to reserve address space for marking stacks"); + return; + } + + // Successfully initialized + _top = _end = ZMarkStackSpaceStart; +} + +bool ZMarkStackSpace::is_initialized() const { + return _top != 0; +} + +bool ZMarkStackSpace::expand() { + const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax; + if (_end + ZMarkStackSpaceExpandSize > max) { + // Expansion limit reached + return false; + } + + void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize, + PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0); + if (res == MAP_FAILED) { + ZErrno err; + log_error(gc, marking)("Failed to map memory for marking stacks (%s)", err.to_string()); + return false; + } + + return true; +} + +uintptr_t ZMarkStackSpace::alloc_space(size_t size) { + uintptr_t top = _top; + + for (;;) { + const uintptr_t new_top = top + size; + if (new_top > _end) { + // Not enough space left + return 0; + } + + const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top); + if (prev_top == top) { + // Success + return top; + } + + // Retry + top = prev_top; + } +} + +uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) { + ZLocker locker(&_expand_lock); + + // Retry allocation before expanding + uintptr_t addr = alloc_space(size); + if (addr != 0) { + return addr; + } + + // Expand stack space + if (!expand()) { + // We currently can't handle the situation where we + // are running out of mark stack space. + fatal("Mark stack overflow (allocated " SIZE_FORMAT "M, size " SIZE_FORMAT "M, max " SIZE_FORMAT "M)," + " use -XX:ZMarkStacksMax=? to increase this limit", + (_end - ZMarkStackSpaceStart) / M, size / M, ZMarkStacksMax / M); + return 0; + } + + log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", + (_end - ZMarkStackSpaceStart) / M, + (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M); + + // Increment top before end to make sure another + // thread can't steal out newly expanded space. + addr = Atomic::add(size, &_top) - size; + _end += ZMarkStackSpaceExpandSize; + + return addr; +} + +uintptr_t ZMarkStackSpace::alloc(size_t size) { + const uintptr_t addr = alloc_space(size); + if (addr != 0) { + return addr; + } + + return expand_and_alloc_space(size); +} + +ZMarkStackAllocator::ZMarkStackAllocator() : + _freelist(), + _space() { + guarantee(sizeof(ZMarkStack) == ZMarkStackSize, "Size mismatch"); + guarantee(sizeof(ZMarkStackMagazine) <= ZMarkStackSize, "Size mismatch"); + + // Prime free list to avoid an immediate space + // expansion when marking starts. + if (_space.is_initialized()) { + prime_freelist(); + } +} + +bool ZMarkStackAllocator::is_initialized() const { + return _space.is_initialized(); +} + +void ZMarkStackAllocator::prime_freelist() { + for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) { + const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize); + ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize); + free_magazine(magazine); + } +} + +ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) { + assert(is_aligned(size, ZMarkStackSize), "Invalid size"); + + // Use first stack as magazine + ZMarkStackMagazine* const magazine = new ((void*)addr) ZMarkStackMagazine(); + for (size_t i = ZMarkStackSize; i < size; i += ZMarkStackSize) { + ZMarkStack* const stack = new ((void*)(addr + i)) ZMarkStack(); + const bool success = magazine->push(stack); + assert(success, "Magazine should never get full"); + } + + return magazine; +} + +ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() { + // Try allocating from the free list first + ZMarkStackMagazine* const magazine = _freelist.pop_atomic(); + if (magazine != NULL) { + return magazine; + } + + // Allocate new magazine + const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize); + if (addr == 0) { + return NULL; + } + + return create_magazine_from_space(addr, ZMarkStackMagazineSize); +} + +void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) { + _freelist.push_atomic(magazine); +} + +ZMarkStripe::ZMarkStripe() : + _published(), + _overflowed() {} + +ZMarkStripeSet::ZMarkStripeSet() : + _nstripes(0), + _nstripes_mask(0), + _stripes() {} + +void ZMarkStripeSet::set_nstripes(size_t nstripes) { + assert(is_power_of_2(nstripes), "Must be a power of two"); + assert(is_power_of_2(ZMarkStripesMax), "Must be a power of two"); + assert(nstripes >= 1, "Invalid number of stripes"); + assert(nstripes <= ZMarkStripesMax, "Invalid number of stripes"); + + _nstripes = nstripes; + _nstripes_mask = nstripes - 1; + + log_debug(gc, marking)("Using " SIZE_FORMAT " mark stripes", _nstripes); +} + +bool ZMarkStripeSet::is_empty() const { + for (size_t i = 0; i < _nstripes; i++) { + if (!_stripes[i].is_empty()) { + return false; + } + } + + return true; +} + +ZMarkStripe* ZMarkStripeSet::stripe_for_worker(uint nworkers, uint worker_id) { + const size_t spillover_limit = (nworkers / _nstripes) * _nstripes; + size_t index; + + if (worker_id < spillover_limit) { + // Not a spillover worker, use natural stripe + index = worker_id & _nstripes_mask; + } else { + // Distribute spillover workers evenly across stripes + const size_t spillover_nworkers = nworkers - spillover_limit; + const size_t spillover_worker_id = worker_id - spillover_limit; + const double spillover_chunk = (double)_nstripes / (double)spillover_nworkers; + index = spillover_worker_id * spillover_chunk; + } + + assert(index < _nstripes, "Invalid index"); + return &_stripes[index]; +} + +ZMarkThreadLocalStacks::ZMarkThreadLocalStacks() : + _magazine(NULL) { + for (size_t i = 0; i < ZMarkStripesMax; i++) { + _stacks[i] = NULL; + } +} + +bool ZMarkThreadLocalStacks::is_empty(const ZMarkStripeSet* stripes) const { + for (size_t i = 0; i < stripes->nstripes(); i++) { + ZMarkStack* const stack = _stacks[i]; + if (stack != NULL) { + return false; + } + } + + return true; +} + +ZMarkStack* ZMarkThreadLocalStacks::allocate_stack(ZMarkStackAllocator* allocator) { + if (_magazine == NULL) { + // Allocate new magazine + _magazine = allocator->alloc_magazine(); + if (_magazine == NULL) { + return NULL; + } + } + + ZMarkStack* stack = NULL; + + if (!_magazine->pop(stack)) { + // Magazine is empty, convert magazine into a new stack + _magazine->~ZMarkStackMagazine(); + stack = new ((void*)_magazine) ZMarkStack(); + _magazine = NULL; + } + + return stack; +} + +void ZMarkThreadLocalStacks::free_stack(ZMarkStackAllocator* allocator, ZMarkStack* stack) { + for (;;) { + if (_magazine == NULL) { + // Convert stack into a new magazine + stack->~ZMarkStack(); + _magazine = new ((void*)stack) ZMarkStackMagazine(); + return; + } + + if (_magazine->push(stack)) { + // Success + return; + } + + // Free and uninstall full magazine + allocator->free_magazine(_magazine); + _magazine = NULL; + } +} + +bool ZMarkThreadLocalStacks::push_slow(ZMarkStackAllocator* allocator, + ZMarkStripe* stripe, + ZMarkStack** stackp, + ZMarkStackEntry entry, + bool publish) { + ZMarkStack* stack = *stackp; + + for (;;) { + if (stack == NULL) { + // Allocate and install new stack + *stackp = stack = allocate_stack(allocator); + if (stack == NULL) { + // Out of mark stack memory + return false; + } + } + + if (stack->push(entry)) { + // Success + return true; + } + + // Publish/Overflow and uninstall stack + stripe->publish_stack(stack, publish); + *stackp = stack = NULL; + } +} + +bool ZMarkThreadLocalStacks::pop_slow(ZMarkStackAllocator* allocator, + ZMarkStripe* stripe, + ZMarkStack** stackp, + ZMarkStackEntry& entry) { + ZMarkStack* stack = *stackp; + + for (;;) { + if (stack == NULL) { + // Try steal and install stack + *stackp = stack = stripe->steal_stack(); + if (stack == NULL) { + // Nothing to steal + return false; + } + } + + if (stack->pop(entry)) { + // Success + return true; + } + + // Free and uninstall stack + free_stack(allocator, stack); + *stackp = stack = NULL; + } +} + +bool ZMarkThreadLocalStacks::flush(ZMarkStackAllocator* allocator, ZMarkStripeSet* stripes) { + bool flushed = false; + + // Flush all stacks + for (size_t i = 0; i < stripes->nstripes(); i++) { + ZMarkStripe* const stripe = stripes->stripe_at(i); + ZMarkStack** const stackp = &_stacks[i]; + ZMarkStack* const stack = *stackp; + if (stack == NULL) { + continue; + } + + // Free/Publish and uninstall stack + if (stack->is_empty()) { + free_stack(allocator, stack); + } else { + stripe->publish_stack(stack); + flushed = true; + } + *stackp = NULL; + } + + return flushed; +} + +void ZMarkThreadLocalStacks::free(ZMarkStackAllocator* allocator) { + // Free and uninstall magazine + if (_magazine != NULL) { + allocator->free_magazine(_magazine); + _magazine = NULL; + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkStack.hpp 2018-06-01 22:30:44.316040862 +0200 @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKSTACK_HPP +#define SHARE_GC_Z_ZMARKSTACK_HPP + +#include "gc/z/zGlobals.hpp" +#include "gc/z/zLock.hpp" +#include "gc/z/zMarkStackEntry.hpp" +#include "memory/allocation.hpp" +#include "utilities/globalDefinitions.hpp" + +template +class ZStack { +private: + size_t _top; + ZStack* _next; + T _slots[S]; + + bool is_full() const; + +public: + ZStack(); + + bool is_empty() const; + + bool push(T value); + bool pop(T& value); + + ZStack* next() const; + ZStack** next_addr(); +}; + +template +class ZStackList { +private: + T* volatile _head; + + T* encode_versioned_pointer(const T* stack, uint32_t version) const; + void decode_versioned_pointer(const T* vstack, T** stack, uint32_t* version) const; + +public: + ZStackList(); + + bool is_empty() const; + + void push_atomic(T* stack); + T* pop_atomic(); +}; + +typedef ZStack ZMarkStack; +typedef ZStackList ZMarkStackList; +typedef ZStack ZMarkStackMagazine; +typedef ZStackList ZMarkStackMagazineList; + +class ZMarkStackSpace { +private: + ZLock _expand_lock; + volatile uintptr_t _top; + volatile uintptr_t _end; + + bool expand(); + + uintptr_t alloc_space(size_t size); + uintptr_t expand_and_alloc_space(size_t size); + +public: + ZMarkStackSpace(); + + bool is_initialized() const; + + uintptr_t alloc(size_t size); +}; + +class ZMarkStackAllocator { +private: + ZMarkStackMagazineList _freelist ATTRIBUTE_ALIGNED(ZCacheLineSize); + ZMarkStackSpace _space ATTRIBUTE_ALIGNED(ZCacheLineSize); + + void prime_freelist(); + ZMarkStackMagazine* create_magazine_from_space(uintptr_t addr, size_t size); + +public: + ZMarkStackAllocator(); + + bool is_initialized() const; + + ZMarkStackMagazine* alloc_magazine(); + void free_magazine(ZMarkStackMagazine* magazine); +}; + +class ZMarkStripe { +private: + ZMarkStackList _published ATTRIBUTE_ALIGNED(ZCacheLineSize); + ZMarkStackList _overflowed ATTRIBUTE_ALIGNED(ZCacheLineSize); + +public: + ZMarkStripe(); + + bool is_empty() const; + + void publish_stack(ZMarkStack* stack, bool publish = true); + ZMarkStack* steal_stack(); +}; + +class ZMarkStripeSet { +private: + size_t _nstripes; + size_t _nstripes_mask; + ZMarkStripe _stripes[ZMarkStripesMax]; + +public: + ZMarkStripeSet(); + + size_t nstripes() const; + void set_nstripes(size_t nstripes); + + bool is_empty() const; + + size_t stripe_id(const ZMarkStripe* stripe) const; + ZMarkStripe* stripe_at(size_t index); + ZMarkStripe* stripe_next(ZMarkStripe* stripe); + ZMarkStripe* stripe_for_worker(uint nworkers, uint worker_id); + ZMarkStripe* stripe_for_addr(uintptr_t addr); +}; + +class ZMarkThreadLocalStacks { +private: + ZMarkStackMagazine* _magazine; + ZMarkStack* _stacks[ZMarkStripesMax]; + + ZMarkStack* allocate_stack(ZMarkStackAllocator* allocator); + void free_stack(ZMarkStackAllocator* allocator, ZMarkStack* stack); + + bool push_slow(ZMarkStackAllocator* allocator, + ZMarkStripe* stripe, + ZMarkStack** stackp, + ZMarkStackEntry entry, + bool publish); + + bool pop_slow(ZMarkStackAllocator* allocator, + ZMarkStripe* stripe, + ZMarkStack** stackp, + ZMarkStackEntry& entry); + +public: + ZMarkThreadLocalStacks(); + + bool is_empty(const ZMarkStripeSet* stripes) const; + + void install(ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStack* stack); + + bool push(ZMarkStackAllocator* allocator, + ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStackEntry entry, + bool publish); + + bool pop(ZMarkStackAllocator* allocator, + ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStackEntry& entry); + + bool flush(ZMarkStackAllocator* allocator, + ZMarkStripeSet* stripes); + + void free(ZMarkStackAllocator* allocator); +}; + +#endif // SHARE_GC_Z_ZMARKSTACK_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkStack.inline.hpp 2018-06-01 22:30:44.701057481 +0200 @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKSTACK_INLINE_HPP +#define SHARE_GC_Z_ZMARKSTACK_INLINE_HPP + +#include "gc/z/zMarkStack.hpp" +#include "utilities/debug.hpp" +#include "runtime/atomic.hpp" + +template +inline ZStack::ZStack() : + _top(0), + _next(NULL) {} + +template +inline bool ZStack::is_empty() const { + return _top == 0; +} + +template +inline bool ZStack::is_full() const { + return _top == S; +} + +template +inline bool ZStack::push(T value) { + if (is_full()) { + return false; + } + + _slots[_top++] = value; + return true; +} + +template +inline bool ZStack::pop(T& value) { + if (is_empty()) { + return false; + } + + value = _slots[--_top]; + return true; +} + +template +inline ZStack* ZStack::next() const { + return _next; +} + +template +inline ZStack** ZStack::next_addr() { + return &_next; +} + +template +inline ZStackList::ZStackList() : + _head(encode_versioned_pointer(NULL, 0)) {} + +template +inline T* ZStackList::encode_versioned_pointer(const T* stack, uint32_t version) const { + uint64_t addr; + + if (stack == NULL) { + addr = (uint32_t)-1; + } else { + addr = ((uint64_t)stack - ZMarkStackSpaceStart) >> ZMarkStackSizeShift; + } + + return (T*)((addr << 32) | (uint64_t)version); +} + +template +inline void ZStackList::decode_versioned_pointer(const T* vstack, T** stack, uint32_t* version) const { + const uint64_t addr = (uint64_t)vstack >> 32; + + if (addr == (uint32_t)-1) { + *stack = NULL; + } else { + *stack = (T*)((addr << ZMarkStackSizeShift) + ZMarkStackSpaceStart); + } + + *version = (uint32_t)(uint64_t)vstack; +} + +template +inline bool ZStackList::is_empty() const { + const T* vstack = _head; + T* stack = NULL; + uint32_t version = 0; + + decode_versioned_pointer(vstack, &stack, &version); + return stack == NULL; +} + +template +inline void ZStackList::push_atomic(T* stack) { + T* vstack = _head; + uint32_t version = 0; + + for (;;) { + decode_versioned_pointer(vstack, stack->next_addr(), &version); + T* const new_vstack = encode_versioned_pointer(stack, version + 1); + T* const prev_vstack = Atomic::cmpxchg(new_vstack, &_head, vstack); + if (prev_vstack == vstack) { + // Success + break; + } + + // Retry + vstack = prev_vstack; + } +} + +template +inline T* ZStackList::pop_atomic() { + T* vstack = _head; + T* stack = NULL; + uint32_t version = 0; + + for (;;) { + decode_versioned_pointer(vstack, &stack, &version); + if (stack == NULL) { + return NULL; + } + + T* const new_vstack = encode_versioned_pointer(stack->next(), version + 1); + T* const prev_vstack = Atomic::cmpxchg(new_vstack, &_head, vstack); + if (prev_vstack == vstack) { + // Success + return stack; + } + + // Retry + vstack = prev_vstack; + } +} + +inline bool ZMarkStripe::is_empty() const { + return _published.is_empty() && _overflowed.is_empty(); +} + +inline void ZMarkStripe::publish_stack(ZMarkStack* stack, bool publish) { + // A stack is published either on the published list or the overflowed + // list. The published list is used by mutators publishing stacks for GC + // workers to work on, while the overflowed list is used by GC workers + // to publish stacks that overflowed. The intention here is to avoid + // contention between mutators and GC workers as much as possible, while + // still allowing GC workers to help out and steal work from each other. + if (publish) { + _published.push_atomic(stack); + } else { + _overflowed.push_atomic(stack); + } +} + +inline ZMarkStack* ZMarkStripe::steal_stack() { + // Steal overflowed stacks first, then published stacks + ZMarkStack* const stack = _overflowed.pop_atomic(); + if (stack != NULL) { + return stack; + } + + return _published.pop_atomic(); +} + +inline size_t ZMarkStripeSet::nstripes() const { + return _nstripes; +} + +inline size_t ZMarkStripeSet::stripe_id(const ZMarkStripe* stripe) const { + const size_t index = ((uintptr_t)stripe - (uintptr_t)_stripes) / sizeof(ZMarkStripe); + assert(index < _nstripes, "Invalid index"); + return index; +} + +inline ZMarkStripe* ZMarkStripeSet::stripe_at(size_t index) { + assert(index < _nstripes, "Invalid index"); + return &_stripes[index]; +} + +inline ZMarkStripe* ZMarkStripeSet::stripe_next(ZMarkStripe* stripe) { + const size_t index = (stripe_id(stripe) + 1) & _nstripes_mask; + assert(index < _nstripes, "Invalid index"); + return &_stripes[index]; +} + +inline ZMarkStripe* ZMarkStripeSet::stripe_for_addr(uintptr_t addr) { + const size_t index = (addr >> ZMarkStripeShift) & _nstripes_mask; + assert(index < _nstripes, "Invalid index"); + return &_stripes[index]; +} + +inline void ZMarkThreadLocalStacks::install(ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStack* stack) { + ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)]; + assert(*stackp == NULL, "Should be empty"); + *stackp = stack; +} + +inline bool ZMarkThreadLocalStacks::push(ZMarkStackAllocator* allocator, + ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStackEntry entry, + bool publish) { + ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)]; + ZMarkStack* const stack = *stackp; + if (stack != NULL && stack->push(entry)) { + return true; + } + + return push_slow(allocator, stripe, stackp, entry, publish); +} + +inline bool ZMarkThreadLocalStacks::pop(ZMarkStackAllocator* allocator, + ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStackEntry& entry) { + ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)]; + ZMarkStack* const stack = *stackp; + if (stack != NULL && stack->pop(entry)) { + return true; + } + + return pop_slow(allocator, stripe, stackp, entry); +} + +#endif // SHARE_GC_Z_ZMARKSTACK_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkStackEntry.hpp 2018-06-01 22:30:45.071073452 +0200 @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKSTACKENTRY_HPP +#define SHARE_GC_Z_ZMARKSTACKENTRY_HPP + +#include "gc/z/zBitField.hpp" +#include "memory/allocation.hpp" + +// +// Mark stack entry layout +// ----------------------- +// +// Object entry +// ------------ +// +// 6 +// 3 2 1 0 +// +---------------------------------------------------------------------+-+-+ +// |11111111 11111111 11111111 11111111 11111111 11111111 11111111 111111|1|1| +// +---------------------------------------------------------------------+-+-+ +// | | | +// | 1-1 Partial Array Flag (1-bit) * | +// | | +// | 0-0 Final Flag (1-bit) * +// | +// * 63-2 Object Address (62-bits) +// +// +// Partial array entry +// ------------------- +// +// 6 3 3 +// 3 2 1 2 1 0 +// +------------------------------------+---------------------------------+-+-+ +// |11111111 11111111 11111111 11111111 |11111111 11111111 11111111 111111|1|1| +// +------------------------------------+---------------------------------+-+-+ +// | | | | +// | | 1-1 Partial Array Flag (1-bit) * | +// | | | +// | | 0-0 Final Flag (1-bit) * +// | | +// | * 31-2 Partial Array Length (30-bits) +// | +// * 63-32 Partial Array Address Offset (32-bits) +// + +class ZMarkStackEntry { +private: + typedef ZBitField field_finalizable; + typedef ZBitField field_partial_array; + typedef ZBitField field_object_address; + typedef ZBitField field_partial_array_length; + typedef ZBitField field_partial_array_offset; + + uint64_t _entry; + +public: + ZMarkStackEntry() { + // This constructor is intentionally left emtpy and does not initialize + // _entry to allow it to be optimized out when instantiating ZMarkStack, + // which has a long array of ZMarkStackEntry elements, but doesn't care + // what _entry is initialized to. + } + + ZMarkStackEntry(uintptr_t object_address, bool finalizable) : + _entry(field_object_address::encode(object_address) | + field_partial_array::encode(false) | + field_finalizable::encode(finalizable)) {} + + ZMarkStackEntry(size_t partial_array_offset, size_t partial_array_length, bool finalizable) : + _entry(field_partial_array_offset::encode(partial_array_offset) | + field_partial_array_length::encode(partial_array_length) | + field_partial_array::encode(true) | + field_finalizable::encode(finalizable)) {} + + bool finalizable() const { + return field_finalizable::decode(_entry); + } + + bool partial_array() const { + return field_partial_array::decode(_entry); + } + + size_t partial_array_offset() const { + return field_partial_array_offset::decode(_entry); + } + + size_t partial_array_length() const { + return field_partial_array_length::decode(_entry); + } + + uintptr_t object_address() const { + return field_object_address::decode(_entry); + } +}; + +#endif // SHARE_GC_Z_ZMARKSTACKENTRY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkTerminate.hpp 2018-06-01 22:30:45.451089856 +0200 @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKTERMINATE_HPP +#define SHARE_GC_Z_ZMARKTERMINATE_HPP + +#include "gc/z/zGlobals.hpp" +#include "memory/allocation.hpp" +#include "utilities/globalDefinitions.hpp" + +class ZMarkTerminate { +private: + uint _nworkers; + volatile uint _nworking_stage0 ATTRIBUTE_ALIGNED(ZCacheLineSize); + volatile uint _nworking_stage1; + + bool enter_stage(volatile uint* nworking_stage); + void exit_stage(volatile uint* nworking_stage); + bool try_exit_stage(volatile uint* nworking_stage); + +public: + ZMarkTerminate(); + + void reset(uint nworkers); + + bool enter_stage0(); + void exit_stage0(); + bool try_exit_stage0(); + + bool enter_stage1(); + bool try_exit_stage1(); +}; + +#endif // SHARE_GC_Z_ZMARKTERMINATE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkTerminate.inline.hpp 2018-06-01 22:30:45.830106215 +0200 @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKTERMINATE_INLINE_HPP +#define SHARE_GC_Z_ZMARKTERMINATE_INLINE_HPP + +#include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" + +inline ZMarkTerminate::ZMarkTerminate() : + _nworkers(0), + _nworking_stage0(0), + _nworking_stage1(0) {} + +inline bool ZMarkTerminate::enter_stage(volatile uint* nworking_stage) { + return Atomic::sub(1u, nworking_stage) == 0; +} + +inline void ZMarkTerminate::exit_stage(volatile uint* nworking_stage) { + Atomic::add(1u, nworking_stage); +} + +inline bool ZMarkTerminate::try_exit_stage(volatile uint* nworking_stage) { + uint nworking = Atomic::load(nworking_stage); + + for (;;) { + if (nworking == 0) { + return false; + } + + const uint new_nworking = nworking + 1; + const uint prev_nworking = Atomic::cmpxchg(new_nworking, nworking_stage, nworking); + if (prev_nworking == nworking) { + // Success + return true; + } + + // Retry + nworking = prev_nworking; + } +} + +inline void ZMarkTerminate::reset(uint nworkers) { + _nworkers = _nworking_stage0 = _nworking_stage1 = nworkers; +} + +inline bool ZMarkTerminate::enter_stage0() { + return enter_stage(&_nworking_stage0); +} + +inline void ZMarkTerminate::exit_stage0() { + exit_stage(&_nworking_stage0); +} + +inline bool ZMarkTerminate::try_exit_stage0() { + return try_exit_stage(&_nworking_stage0); +} + +inline bool ZMarkTerminate::enter_stage1() { + return enter_stage(&_nworking_stage1); +} + +inline bool ZMarkTerminate::try_exit_stage1() { + return try_exit_stage(&_nworking_stage1); +} + +#endif // SHARE_GC_Z_ZMARKTERMINATE_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMemory.cpp 2018-06-01 22:30:46.215122834 +0200 @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zList.inline.hpp" +#include "gc/z/zMemory.inline.hpp" +#include "memory/allocation.inline.hpp" + +uintptr_t ZMemoryManager::alloc_from_front(size_t size) { + ZListIterator iter(&_freelist); + for (ZMemory* area; iter.next(&area);) { + if (area->size() >= size) { + if (area->size() == size) { + // Exact match, remove area + const uintptr_t start = area->start(); + _freelist.remove(area); + delete area; + return start; + } else { + // Larger than requested, shrink area + const uintptr_t start = area->start(); + area->shrink_from_front(size); + return start; + } + } + } + + // Out of memory + return UINTPTR_MAX; +} + +uintptr_t ZMemoryManager::alloc_from_back(size_t size) { + ZListReverseIterator iter(&_freelist); + for (ZMemory* area; iter.next(&area);) { + if (area->size() >= size) { + if (area->size() == size) { + // Exact match, remove area + const uintptr_t start = area->start(); + _freelist.remove(area); + delete area; + return start; + } else { + // Larger than requested, shrink area + area->shrink_from_back(size); + return area->end(); + } + } + } + + // Out of memory + return UINTPTR_MAX; +} + +void ZMemoryManager::free(uintptr_t start, size_t size) { + assert(start != UINTPTR_MAX, "Invalid address"); + const uintptr_t end = start + size; + + ZListIterator iter(&_freelist); + for (ZMemory* area; iter.next(&area);) { + if (start < area->start()) { + ZMemory* const prev = _freelist.prev(area); + if (prev != NULL && start == prev->end()) { + if (end == area->start()) { + // Merge with prev and current area + prev->grow_from_back(size + area->size()); + _freelist.remove(area); + delete area; + } else { + // Merge with prev area + prev->grow_from_back(size); + } + } else if (end == area->start()) { + // Merge with current area + area->grow_from_front(size); + } else { + // Insert new area before current area + assert(end < area->start(), "Areas must not overlap"); + ZMemory* new_area = new ZMemory(start, size); + _freelist.insert_before(area, new_area); + } + + // Done + return; + } + } + + // Insert last + ZMemory* const last = _freelist.last(); + if (last != NULL && start == last->end()) { + // Merge with last area + last->grow_from_back(size); + } else { + // Insert new area last + ZMemory* new_area = new ZMemory(start, size); + _freelist.insert_last(new_area); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMemory.hpp 2018-06-01 22:30:46.595139237 +0200 @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMEMORY_HPP +#define SHARE_GC_Z_ZMEMORY_HPP + +#include "gc/z/zList.hpp" +#include "memory/allocation.hpp" + +class ZMemory : public CHeapObj { + friend class ZList; + +private: + uintptr_t _start; + uintptr_t _end; + ZListNode _node; + +public: + ZMemory(uintptr_t start, size_t size); + + uintptr_t start() const; + uintptr_t end() const; + size_t size() const; + + void shrink_from_front(size_t size); + void shrink_from_back(size_t size); + void grow_from_front(size_t size); + void grow_from_back(size_t size); +}; + +class ZMemoryManager { +private: + ZList _freelist; + +public: + uintptr_t alloc_from_front(size_t size); + uintptr_t alloc_from_back(size_t size); + void free(uintptr_t start, size_t size); +}; + +#endif // SHARE_GC_Z_ZMEMORY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMemory.inline.hpp 2018-06-01 22:30:46.980155856 +0200 @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMEMORY_INLINE_HPP +#define SHARE_GC_Z_ZMEMORY_INLINE_HPP + +#include "gc/z/zMemory.hpp" +#include "utilities/debug.hpp" + +inline ZMemory::ZMemory(uintptr_t start, size_t size) : + _start(start), + _end(start + size) {} + +inline uintptr_t ZMemory::start() const { + return _start; +} + +inline uintptr_t ZMemory::end() const { + return _end; +} + +inline size_t ZMemory::size() const { + return end() - start(); +} + +inline void ZMemory::shrink_from_front(size_t size) { + assert(this->size() > size, "Too small"); + _start += size; +} + +inline void ZMemory::shrink_from_back(size_t size) { + assert(this->size() > size, "Too small"); + _end -= size; +} + +inline void ZMemory::grow_from_front(size_t size) { + assert(start() >= size, "Too big"); + _start -= size; +} + +inline void ZMemory::grow_from_back(size_t size) { + _end += size; +} + +#endif // SHARE_GC_Z_ZMEMORY_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMessagePort.hpp 2018-06-01 22:30:47.357172130 +0200 @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMESSAGEPORT_HPP +#define SHARE_GC_Z_ZMESSAGEPORT_HPP + +#include "gc/z/zFuture.hpp" +#include "gc/z/zList.hpp" +#include "memory/allocation.hpp" +#include "runtime/mutex.hpp" + +template class ZMessageRequest; + +template +class ZMessagePort { +private: + typedef ZMessageRequest Request; + + Monitor _monitor; + bool _has_message; + T _message; + uint64_t _seqnum; + ZList _queue; + +public: + ZMessagePort(); + + void send_sync(T message); + void send_async(T message); + + T receive(); + void ack(); +}; + +class ZRendezvousPort { +private: + ZMessagePort _port; + +public: + void signal(); + void wait(); + void ack(); +}; + +#endif // SHARE_GC_Z_ZMESSAGEPORT_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMessagePort.inline.hpp 2018-06-01 22:30:47.731188274 +0200 @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMESSAGEPORT_INLINE_HPP +#define SHARE_GC_Z_ZMESSAGEPORT_INLINE_HPP + +#include "gc/z/zMessagePort.hpp" +#include "gc/z/zFuture.inline.hpp" +#include "gc/z/zList.inline.hpp" +#include "runtime/mutexLocker.hpp" + +template +class ZMessageRequest : public StackObj { + friend class ZList; + +private: + T _message; + uint64_t _seqnum; + ZFuture _result; + ZListNode _node; + +public: + void initialize(T message, uint64_t seqnum) { + _message = message; + _seqnum = seqnum; + } + + T message() const { + return _message; + } + + uint64_t seqnum() const { + return _seqnum; + } + + void wait() { + const T message = _result.get(); + assert(message == _message, "Message mismatch"); + } + + void satisfy(T message) { + _result.set(message); + } +}; + +template +inline ZMessagePort::ZMessagePort() : + _monitor(Monitor::leaf, + "ZMessagePort", + Monitor::_allow_vm_block_flag, + Monitor::_safepoint_check_never), + _has_message(false), + _seqnum(0), + _queue() {} + +template +inline void ZMessagePort::send_sync(T message) { + Request request; + + { + // Enqueue message + MonitorLockerEx ml(&_monitor, Monitor::_no_safepoint_check_flag); + request.initialize(message, _seqnum); + _queue.insert_last(&request); + ml.notify(); + } + + // Wait for completion + request.wait(); +} + +template +inline void ZMessagePort::send_async(T message) { + MonitorLockerEx ml(&_monitor, Monitor::_no_safepoint_check_flag); + if (!_has_message) { + // Post message + _message = message; + _has_message = true; + ml.notify(); + } +} + +template +inline T ZMessagePort::receive() { + MonitorLockerEx ml(&_monitor, Monitor::_no_safepoint_check_flag); + + // Wait for message + while (!_has_message && _queue.is_empty()) { + ml.wait(Monitor::_no_safepoint_check_flag); + } + + // Increment request sequence number + _seqnum++; + + if (!_has_message) { + // Message available in the queue + _message = _queue.first()->message(); + _has_message = true; + } + + return _message; +} + +template +inline void ZMessagePort::ack() { + MonitorLockerEx ml(&_monitor, Monitor::_no_safepoint_check_flag); + + if (!_has_message) { + // Nothing to ack + return; + } + + // Satisfy requests (and duplicates) in queue + ZListIterator iter(&_queue); + for (Request* request; iter.next(&request);) { + if (request->message() == _message && request->seqnum() < _seqnum) { + // Dequeue and satisfy request. Note that the dequeue operation must + // happen first, since the request will immediately be deallocated + // once it has been satisfied. + _queue.remove(request); + request->satisfy(_message); + } + } + + if (_queue.is_empty()) { + // Queue is empty + _has_message = false; + } else { + // Post first message in queue + _message = _queue.first()->message(); + } +} + +inline void ZRendezvousPort::signal() { + _port.send_sync(true /* ignored */); +} + +inline void ZRendezvousPort::wait() { + _port.receive(); +} + +inline void ZRendezvousPort::ack() { + _port.ack(); +} + +#endif // SHARE_GC_Z_ZMESSAGEPORT_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMetronome.cpp 2018-06-01 22:30:48.105204418 +0200 @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zMetronome.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/timer.hpp" +#include "utilities/ticks.hpp" + +ZMetronome::ZMetronome(uint64_t hz) : + _monitor(Monitor::leaf, "ZMetronome", false, Monitor::_safepoint_check_never), + _interval_ms(MILLIUNITS / hz), + _start_ms(0), + _nticks(0), + _stopped(false) {} + +uint64_t ZMetronome::nticks() const { + return _nticks; +} + +bool ZMetronome::wait_for_tick() { + if (_nticks++ == 0) { + // First tick, set start time + const Ticks now = Ticks::now(); + _start_ms = TimeHelper::counter_to_millis(now.value()); + } + + for (;;) { + // We might wake up spuriously from wait, so always recalculate + // the timeout after a wakeup to see if we need to wait again. + const Ticks now = Ticks::now(); + const uint64_t now_ms = TimeHelper::counter_to_millis(now.value()); + const uint64_t next_ms = _start_ms + (_interval_ms * _nticks); + const int64_t timeout_ms = next_ms - now_ms; + + MonitorLockerEx ml(&_monitor, Monitor::_no_safepoint_check_flag); + if (!_stopped && timeout_ms > 0) { + // Wait + ml.wait(Monitor::_no_safepoint_check_flag, timeout_ms); + } else { + // Tick + return !_stopped; + } + } +} + +void ZMetronome::stop() { + MonitorLockerEx ml(&_monitor, Monitor::_no_safepoint_check_flag); + _stopped = true; + ml.notify(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMetronome.hpp 2018-06-01 22:30:48.492221123 +0200 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMETRONOME_HPP +#define SHARE_GC_Z_ZMETRONOME_HPP + +#include "memory/allocation.hpp" +#include "runtime/mutex.hpp" + +class ZMetronome : public StackObj { +private: + Monitor _monitor; + const uint64_t _interval_ms; + uint64_t _start_ms; + uint64_t _nticks; + bool _stopped; + +public: + ZMetronome(uint64_t hz); + + uint64_t nticks() const; + bool wait_for_tick(); + void stop(); +}; + +#endif // SHARE_GC_Z_ZMETRONOME_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zNMethodTable.cpp 2018-06-01 22:30:48.876237699 +0200 @@ -0,0 +1,465 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "code/relocInfo.hpp" +#include "code/nativeInst.hpp" +#include "code/nmethod.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHash.inline.hpp" +#include "gc/z/zNMethodTable.hpp" +#include "logging/log.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" + +class ZNMethodWithImmediateOops { +private: + nmethod* const _nm; + const size_t _nimmediate_oops; + + static size_t header_size(); + + ZNMethodWithImmediateOops(nmethod* nm, const GrowableArray& immediate_oops); + +public: + static ZNMethodWithImmediateOops* create(nmethod* nm, const GrowableArray& immediate_oops); + static void destroy(ZNMethodWithImmediateOops* nmi); + + nmethod* method() const; + size_t immediate_oops_count() const; + oop** immediate_oops_begin() const; + oop** immediate_oops_begin_safe() const; + oop** immediate_oops_end() const; +}; + +size_t ZNMethodWithImmediateOops::header_size() { + const size_t size = sizeof(ZNMethodWithImmediateOops); + assert(is_aligned(size, sizeof(oop*)), "Header misaligned"); + return size; +} + +ZNMethodWithImmediateOops::ZNMethodWithImmediateOops(nmethod* nm, const GrowableArray& immediate_oops) : + _nm(nm), + _nimmediate_oops(immediate_oops.length()) { + // Save all immediate oops + for (size_t i = 0; i < _nimmediate_oops; i++) { + immediate_oops_begin()[i] = immediate_oops.at(i); + } +} + +ZNMethodWithImmediateOops* ZNMethodWithImmediateOops::create(nmethod* nm, const GrowableArray& immediate_oops) { + // Allocate memory for the ZNMethodWithImmediateOops object + // plus the immediate oop* array that follows right after. + const size_t size = header_size() + (sizeof(oop*) * immediate_oops.length()); + void* const method_with_immediate_oops = NEW_C_HEAP_ARRAY(uint8_t, size, mtGC); + return ::new (method_with_immediate_oops) ZNMethodWithImmediateOops(nm, immediate_oops); +} + +void ZNMethodWithImmediateOops::destroy(ZNMethodWithImmediateOops* nmi) { + FREE_C_HEAP_ARRAY(uint8_t, nmi); +} + +nmethod* ZNMethodWithImmediateOops::method() const { + return _nm; +} + +size_t ZNMethodWithImmediateOops::immediate_oops_count() const { + return _nimmediate_oops; +} + +oop** ZNMethodWithImmediateOops::immediate_oops_begin() const { + // The immediate oop* array starts immediately after this object + return (oop**)((uintptr_t)this + header_size()); +} + +oop** ZNMethodWithImmediateOops::immediate_oops_begin_safe() const { + // Non-entrant nmethods have a jump instruction patched into the beginning + // of the verified entry point, which could have overwritten an immediate + // oop. If so, make sure we skip over that oop. + if (_nm->is_not_entrant()) { + oop* const first_immediate_oop = *immediate_oops_begin(); + oop* const safe_begin = (oop*)(_nm->verified_entry_point() + NativeJump::instruction_size); + if (first_immediate_oop < safe_begin) { + // First immediate oop overwritten, skip it + return immediate_oops_begin() + 1; + } + } + + // First immediate oop not overwritten + return immediate_oops_begin(); +} + + +oop** ZNMethodWithImmediateOops::immediate_oops_end() const { + return immediate_oops_begin() + immediate_oops_count(); +} + +ZNMethodTableEntry* ZNMethodTable::_table = NULL; +size_t ZNMethodTable::_size = 0; +size_t ZNMethodTable::_nregistered = 0; +size_t ZNMethodTable::_nunregistered = 0; +volatile size_t ZNMethodTable::_claimed = 0; + +ZNMethodTableEntry ZNMethodTable::create_entry(nmethod* nm) { + GrowableArray immediate_oops; + bool non_immediate_oops = false; + + // Find all oops relocations + RelocIterator iter(nm); + while (iter.next()) { + if (iter.type() != relocInfo::oop_type) { + // Not an oop + continue; + } + + oop_Relocation* r = iter.oop_reloc(); + + if (!r->oop_is_immediate()) { + // Non-immediate oop found + non_immediate_oops = true; + continue; + } + + if (r->oop_value() != NULL) { + // Non-NULL immediate oop found. NULL oops can safely be + // ignored since the method will be re-registered if they + // are later patched to be non-NULL. + immediate_oops.push(r->oop_addr()); + } + } + + // oops_count() returns the number of oops in the oop table plus one + if (immediate_oops.is_empty() && nm->oops_count() == 1) { + // No oops found, return empty entry + return ZNMethodTableEntry(); + } + + if (immediate_oops.is_empty()) { + // No immediate oops found, return entry without immediate oops + return ZNMethodTableEntry(nm, non_immediate_oops); + } + + // Return entry with immediate oops + return ZNMethodTableEntry(ZNMethodWithImmediateOops::create(nm, immediate_oops), non_immediate_oops); +} + +void ZNMethodTable::destroy_entry(ZNMethodTableEntry entry) { + if (entry.immediate_oops()) { + ZNMethodWithImmediateOops::destroy(entry.method_with_immediate_oops()); + } +} + +nmethod* ZNMethodTable::method(ZNMethodTableEntry entry) { + return entry.immediate_oops() ? entry.method_with_immediate_oops()->method() : entry.method(); +} + +size_t ZNMethodTable::first_index(const nmethod* nm, size_t size) { + assert(is_power_of_2(size), "Invalid size"); + const size_t mask = size - 1; + const size_t hash = ZHash::address_to_uint32((uintptr_t)nm); + return hash & mask; +} + +size_t ZNMethodTable::next_index(size_t prev_index, size_t size) { + assert(is_power_of_2(size), "Invalid size"); + const size_t mask = size - 1; + return (prev_index + 1) & mask; +} + +bool ZNMethodTable::register_entry(ZNMethodTableEntry* table, size_t size, ZNMethodTableEntry entry) { + const nmethod* const nm = method(entry); + size_t index = first_index(nm, size); + + for (;;) { + const ZNMethodTableEntry table_entry = table[index]; + + if (!table_entry.registered() && !table_entry.unregistered()) { + // Insert new entry + table[index] = entry; + return true; + } + + if (table_entry.registered() && method(table_entry) == nm) { + // Replace existing entry + destroy_entry(table_entry); + table[index] = entry; + return false; + } + + index = next_index(index, size); + } +} + +bool ZNMethodTable::unregister_entry(ZNMethodTableEntry* table, size_t size, const nmethod* nm) { + if (size == 0) { + // Table is empty + return false; + } + + size_t index = first_index(nm, size); + + for (;;) { + const ZNMethodTableEntry table_entry = table[index]; + + if (!table_entry.registered() && !table_entry.unregistered()) { + // Entry not found + return false; + } + + if (table_entry.registered() && method(table_entry) == nm) { + // Remove entry + destroy_entry(table_entry); + table[index] = ZNMethodTableEntry(true /* unregistered */); + return true; + } + + index = next_index(index, size); + } +} + +void ZNMethodTable::rebuild(size_t new_size) { + assert(is_power_of_2(new_size), "Invalid size"); + + log_debug(gc, nmethod)("Rebuilding NMethod Table: " + SIZE_FORMAT "->" SIZE_FORMAT " entries, " + SIZE_FORMAT "(%.0lf%%->%.0lf%%) registered, " + SIZE_FORMAT "(%.0lf%%->%.0lf%%) unregistered", + _size, new_size, + _nregistered, percent_of(_nregistered, _size), percent_of(_nregistered, new_size), + _nunregistered, percent_of(_nunregistered, _size), 0.0); + + // Allocate new table + ZNMethodTableEntry* const new_table = new ZNMethodTableEntry[new_size]; + + // Transfer all registered entries + for (size_t i = 0; i < _size; i++) { + const ZNMethodTableEntry entry = _table[i]; + if (entry.registered()) { + register_entry(new_table, new_size, entry); + } + } + + // Delete old table + delete [] _table; + + // Install new table + _table = new_table; + _size = new_size; + _nunregistered = 0; +} + +void ZNMethodTable::rebuild_if_needed() { + // The hash table uses linear probing. To avoid wasting memory while + // at the same time maintaining good hash collision behavior we want + // to keep the table occupancy between 30% and 70%. The table always + // grows/shrinks by doubling/halving its size. Pruning of unregistered + // entries is done by rebuilding the table with or without resizing it. + const size_t min_size = 1024; + const size_t shrink_threshold = _size * 0.30; + const size_t prune_threshold = _size * 0.65; + const size_t grow_threshold = _size * 0.70; + + if (_size == 0) { + // Initialize table + rebuild(min_size); + } else if (_nregistered < shrink_threshold && _size > min_size) { + // Shrink table + rebuild(_size / 2); + } else if (_nregistered + _nunregistered > grow_threshold) { + // Prune or grow table + if (_nregistered < prune_threshold) { + // Prune table + rebuild(_size); + } else { + // Grow table + rebuild(_size * 2); + } + } +} + +void ZNMethodTable::log_register(const nmethod* nm, ZNMethodTableEntry entry) { + LogTarget(Trace, gc, nmethod) log; + if (!log.is_enabled()) { + return; + } + + log.print("Register NMethod: %s.%s (" PTR_FORMAT "), " + "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s", + nm->method()->method_holder()->external_name(), + nm->method()->name()->as_C_string(), + p2i(nm), + nm->compiler_name(), + nm->oops_count() - 1, + entry.immediate_oops() ? entry.method_with_immediate_oops()->immediate_oops_count() : 0, + BOOL_TO_STR(entry.non_immediate_oops())); + + LogTarget(Trace, gc, nmethod, oops) log_oops; + if (!log_oops.is_enabled()) { + return; + } + + // Print nmethod oops table + oop* const begin = nm->oops_begin(); + oop* const end = nm->oops_end(); + for (oop* p = begin; p < end; p++) { + log_oops.print(" Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)", + (p - begin), p2i(*p), (*p)->klass()->external_name()); + } + + if (entry.immediate_oops()) { + // Print nmethod immediate oops + const ZNMethodWithImmediateOops* const nmi = entry.method_with_immediate_oops(); + oop** const begin = nmi->immediate_oops_begin(); + oop** const end = nmi->immediate_oops_end(); + for (oop** p = begin; p < end; p++) { + log_oops.print(" ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)", + (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name()); + } + } +} + +void ZNMethodTable::log_unregister(const nmethod* nm) { + LogTarget(Debug, gc, nmethod) log; + if (!log.is_enabled()) { + return; + } + + log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")", + nm->method()->method_holder()->external_name(), + nm->method()->name()->as_C_string(), + p2i(nm)); +} + +size_t ZNMethodTable::registered_nmethods() { + return _nregistered; +} + +size_t ZNMethodTable::unregistered_nmethods() { + return _nunregistered; +} + +void ZNMethodTable::register_nmethod(nmethod* nm) { + ResourceMark rm; + + // Create entry + const ZNMethodTableEntry entry = create_entry(nm); + + log_register(nm, entry); + + if (!entry.registered()) { + // Method doesn't have any oops, ignore it + return; + } + + // Grow/Shrink/Prune table if needed + rebuild_if_needed(); + + // Insert new entry + if (register_entry(_table, _size, entry)) { + // New entry registered. When register_entry() instead returns + // false the nmethod was already in the table so we do not want + // to increase number of registered entries in that case. + _nregistered++; + } +} + +void ZNMethodTable::unregister_nmethod(nmethod* nm) { + ResourceMark rm; + + log_unregister(nm); + + // Remove entry + if (unregister_entry(_table, _size, nm)) { + // Entry was unregistered. When unregister_entry() instead returns + // false the nmethod was not in the table (because it didn't have + // any oops) so we do not want to decrease the number of registered + // entries in that case. + _nregistered--; + _nunregistered++; + } +} + +void ZNMethodTable::gc_prologue() { + _claimed = 0; +} + +void ZNMethodTable::gc_epilogue() { + assert(_claimed >= _size, "Failed to claim all table entries"); +} + +void ZNMethodTable::entry_oops_do(ZNMethodTableEntry entry, OopClosure* cl) { + nmethod* const nm = method(entry); + if (!nm->is_alive()) { + // No need to visit oops + return; + } + + // Process oops table + oop* const begin = nm->oops_begin(); + oop* const end = nm->oops_end(); + for (oop* p = begin; p < end; p++) { + if (*p != Universe::non_oop_word()) { + cl->do_oop(p); + } + } + + if (entry.immediate_oops()) { + // Process immediate oops + const ZNMethodWithImmediateOops* const nmi = entry.method_with_immediate_oops(); + oop** const begin = nmi->immediate_oops_begin_safe(); + oop** const end = nmi->immediate_oops_end(); + for (oop** p = begin; p < end; p++) { + cl->do_oop(*p); + } + } + + if (entry.non_immediate_oops()) { + // Process non-immediate oops + nm->fix_oop_relocations(); + } +} + +void ZNMethodTable::oops_do(OopClosure* cl) { + for (;;) { + // Claim table partition. Each partition is currently sized to span + // two cache lines. This number is just a guess, but seems to work well. + const size_t partition_size = (ZCacheLineSize * 2) / sizeof(ZNMethodTableEntry); + const size_t partition_start = MIN2(Atomic::add(partition_size, &_claimed) - partition_size, _size); + const size_t partition_end = MIN2(partition_start + partition_size, _size); + if (partition_start == partition_end) { + // End of table + break; + } + + // Process table partition + for (size_t i = partition_start; i < partition_end; i++) { + const ZNMethodTableEntry entry = _table[i]; + if (entry.registered()) { + entry_oops_do(entry, cl); + } + } + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zNMethodTable.hpp 2018-06-01 22:30:49.250253843 +0200 @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZNMETHODTABLE_HPP +#define SHARE_GC_Z_ZNMETHODTABLE_HPP + +#include "gc/z/zGlobals.hpp" +#include "gc/z/zNMethodTableEntry.hpp" +#include "memory/allocation.hpp" + +class ZNMethodTable : public AllStatic { +private: + static ZNMethodTableEntry* _table; + static size_t _size; + static size_t _nregistered; + static size_t _nunregistered; + static volatile size_t _claimed ATTRIBUTE_ALIGNED(ZCacheLineSize); + + static ZNMethodTableEntry create_entry(nmethod* nm); + static void destroy_entry(ZNMethodTableEntry entry); + + static nmethod* method(ZNMethodTableEntry entry); + + static size_t first_index(const nmethod* nm, size_t size); + static size_t next_index(size_t prev_index, size_t size); + + static bool register_entry(ZNMethodTableEntry* table, size_t size, ZNMethodTableEntry entry); + static bool unregister_entry(ZNMethodTableEntry* table, size_t size, const nmethod* nm); + + static void rebuild(size_t new_size); + static void rebuild_if_needed(); + + static void log_register(const nmethod* nm, ZNMethodTableEntry entry); + static void log_unregister(const nmethod* nm); + + static void entry_oops_do(ZNMethodTableEntry entry, OopClosure* cl); + +public: + static size_t registered_nmethods(); + static size_t unregistered_nmethods(); + + static void register_nmethod(nmethod* nm); + static void unregister_nmethod(nmethod* nm); + + static void gc_prologue(); + static void gc_epilogue(); + + static void oops_do(OopClosure* cl); +}; + +#endif // SHARE_GC_Z_ZNMETHODTABLE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zNMethodTableEntry.hpp 2018-06-01 22:30:49.631270289 +0200 @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZNMETHODTABLEENTRY_HPP +#define SHARE_GC_Z_ZNMETHODTABLEENTRY_HPP + +#include "gc/z/zBitField.hpp" +#include "memory/allocation.hpp" + +// +// NMethod table entry layout +// -------------------------- +// +// 6 +// 3 3 2 1 0 +// +--------------------------------------------------------------------+-+-+-+ +// |11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111|1|1|1| +// +--------------------------------------------------------------------+-+-+-+ +// | | | | +// | 2-2 Non-immediate Oops Flag (1-bits) * | | +// | | | +// | 1-1 Immediate Oops/Unregistered Flag (1-bits) * | +// | | +// | 0-0 Registered Flag (1-bits) * +// | +// * 63-3 NMethod/ZNMethodWithImmediateOops Address (61-bits) +// + +class nmethod; +class ZNMethodWithImmediateOops; + +class ZNMethodTableEntry : public CHeapObj { +private: + typedef ZBitField field_registered; + typedef ZBitField field_unregistered; + typedef ZBitField field_immediate_oops; + typedef ZBitField field_non_immediate_oops; + typedef ZBitField field_method; + typedef ZBitField field_method_with_immediate_oops; + + uint64_t _entry; + +public: + ZNMethodTableEntry(bool unregistered = false) : + _entry(field_unregistered::encode(unregistered) | + field_registered::encode(false)) {} + + ZNMethodTableEntry(nmethod* method, bool non_immediate_oops) : + _entry(field_method::encode(method) | + field_non_immediate_oops::encode(non_immediate_oops) | + field_immediate_oops::encode(false) | + field_registered::encode(true)) {} + + ZNMethodTableEntry(ZNMethodWithImmediateOops* method_with_immediate_oops, bool non_immediate_oops) : + _entry(field_method_with_immediate_oops::encode(method_with_immediate_oops) | + field_non_immediate_oops::encode(non_immediate_oops) | + field_immediate_oops::encode(true) | + field_registered::encode(true)) {} + + bool registered() const { + return field_registered::decode(_entry); + } + + bool unregistered() const { + return field_unregistered::decode(_entry); + } + + bool immediate_oops() const { + return field_immediate_oops::decode(_entry); + } + + bool non_immediate_oops() const { + return field_non_immediate_oops::decode(_entry); + } + + nmethod* method() const { + return field_method::decode(_entry); + } + + ZNMethodWithImmediateOops* method_with_immediate_oops() const { + return field_method_with_immediate_oops::decode(_entry); + } +}; + +#endif // SHARE_GC_Z_ZNMETHODTABLEENTRY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zNUMA.cpp 2018-06-01 22:30:50.014286822 +0200 @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "gc/z/zNUMA.hpp" +#include "logging/log.hpp" +#include "runtime/os.hpp" + +bool ZNUMA::_enabled; + +void ZNUMA::initialize() { + initialize_platform(); + + log_info(gc, init)("NUMA Support: %s", to_string()); + if (is_enabled()) { + log_info(gc, init)("NUMA Nodes: %u", count()); + } +} + +bool ZNUMA::is_enabled() { + return _enabled; +} + +void ZNUMA::memory_interleave(uintptr_t addr, size_t size) { + if (!_enabled) { + // NUMA support not enabled + return; + } + + os::numa_make_global((char*)addr, size); +} + +const char* ZNUMA::to_string() { + return _enabled ? "Enabled" : "Disabled"; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zNUMA.hpp 2018-06-01 22:30:50.394303225 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZNUMA_HPP +#define SHARE_GC_Z_ZNUMA_HPP + +#include "memory/allocation.hpp" + +class ZNUMA : public AllStatic { +private: + static bool _enabled; + + static void initialize_platform(); + +public: + static void initialize(); + static bool is_enabled(); + + static uint32_t count(); + static uint32_t id(); + + static uint32_t memory_id(uintptr_t addr); + static void memory_interleave(uintptr_t addr, size_t size); + + static const char* to_string(); +}; + +#endif // SHARE_GC_Z_ZNUMA_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zObjectAllocator.cpp 2018-06-01 22:30:50.783320017 +0200 @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/threadLocalAllocBuffer.inline.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zObjectAllocator.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zThread.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "logging/log.hpp" +#include "runtime/atomic.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/thread.hpp" +#include "runtime/threadSMR.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +static const ZStatCounter ZCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", ZStatUnitOpsPerSecond); +static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond); +static const ZStatSubPhase ZSubPhasePauseRetireTLABS("Pause Retire TLABS"); +static const ZStatSubPhase ZSubPhasePauseRemapTLABS("Pause Remap TLABS"); + +ZObjectAllocator::ZObjectAllocator(uint nworkers) : + _nworkers(nworkers), + _used(0), + _shared_medium_page(NULL), + _shared_small_page(NULL), + _worker_small_page(NULL) {} + +ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) { + ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags); + if (page != NULL) { + // Increment used bytes + Atomic::add(size, _used.addr()); + } + + return page; +} + +uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page, + uint8_t page_type, + size_t page_size, + size_t size, + ZAllocationFlags flags) { + uintptr_t addr = 0; + ZPage* page = *shared_page; + + if (page != NULL) { + addr = page->alloc_object_atomic(size); + } + + if (addr == 0) { + // Allocate new page + ZPage* const new_page = alloc_page(page_type, page_size, flags); + if (new_page != NULL) { + // Allocate object before installing the new page + addr = new_page->alloc_object(size); + + retry: + // Install new page + ZPage* const prev_page = Atomic::cmpxchg(new_page, shared_page, page); + if (prev_page != page) { + if (prev_page == NULL) { + // Previous page was retired, retry installing the new page + page = prev_page; + goto retry; + } + + // Another page already installed, try allocation there first + const uintptr_t prev_addr = prev_page->alloc_object_atomic(size); + if (prev_addr == 0) { + // Allocation failed, retry installing the new page + page = prev_page; + goto retry; + } + + // Allocation succeeded in already installed page + addr = prev_addr; + + // Undo new page allocation + ZHeap::heap()->undo_alloc_page(new_page); + } + } + } + + return addr; +} + +uintptr_t ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags flags) { + assert(ZThread::is_java(), "Should be a Java thread"); + + uintptr_t addr = 0; + + // Allocate new large page + const size_t page_size = align_up(size, ZPageSizeMin); + ZPage* const page = alloc_page(ZPageTypeLarge, page_size, flags); + if (page != NULL) { + // Allocate the object + addr = page->alloc_object(size); + } + + return addr; +} + +uintptr_t ZObjectAllocator::alloc_medium_object(size_t size, ZAllocationFlags flags) { + return alloc_object_in_shared_page(_shared_medium_page.addr(), ZPageTypeMedium, ZPageSizeMedium, size, flags); +} + +uintptr_t ZObjectAllocator::alloc_small_object_from_nonworker(size_t size, ZAllocationFlags flags) { + assert(ZThread::is_java() || ZThread::is_vm(), "Should be a Java or VM thread"); + + if (flags.relocation() && flags.java_thread()) { + // For relocations from Java threads, try TLAB allocation first + const uintptr_t addr = (uintptr_t)Thread::current()->tlab().allocate(ZUtils::bytes_to_words(size)); + if (addr != 0) { + return addr; + } + } + + // Non-worker small page allocation can never use the reserve + flags.set_no_reserve(); + + return alloc_object_in_shared_page(_shared_small_page.addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags); +} + +uintptr_t ZObjectAllocator::alloc_small_object_from_worker(size_t size, ZAllocationFlags flags) { + assert(ZThread::is_worker(), "Should be a worker thread"); + + ZPage* page = _worker_small_page.get(); + uintptr_t addr = 0; + + if (page != NULL) { + addr = page->alloc_object(size); + } + + if (addr == 0) { + // Allocate new page + page = alloc_page(ZPageTypeSmall, ZPageSizeSmall, flags); + if (page != NULL) { + addr = page->alloc_object(size); + } + _worker_small_page.set(page); + } + + return addr; +} + +uintptr_t ZObjectAllocator::alloc_small_object(size_t size, ZAllocationFlags flags) { + if (flags.worker_thread()) { + return alloc_small_object_from_worker(size, flags); + } else { + return alloc_small_object_from_nonworker(size, flags); + } +} + +uintptr_t ZObjectAllocator::alloc_object(size_t size, ZAllocationFlags flags) { + if (size <= ZObjectSizeLimitSmall) { + // Small + return alloc_small_object(size, flags); + } else if (size <= ZObjectSizeLimitMedium) { + // Medium + return alloc_medium_object(size, flags); + } else { + // Large + return alloc_large_object(size, flags); + } +} + +uintptr_t ZObjectAllocator::alloc_object(size_t size) { + assert(ZThread::is_java(), "Must be a Java thread"); + + ZAllocationFlags flags; + flags.set_java_thread(); + flags.set_no_reserve(); + + if (!ZStallOnOutOfMemory) { + flags.set_non_blocking(); + } + + return alloc_object(size, flags); +} + +uintptr_t ZObjectAllocator::alloc_object_for_relocation(size_t size) { + assert(ZThread::is_java() || ZThread::is_worker() || ZThread::is_vm(), "Unknown thread"); + + ZAllocationFlags flags; + flags.set_relocation(); + flags.set_non_blocking(); + + if (ZThread::is_worker()) { + flags.set_worker_thread(); + } else if (ZThread::is_java()) { + flags.set_java_thread(); + } + + return alloc_object(size, flags); +} + +bool ZObjectAllocator::undo_alloc_large_object(ZPage* page) { + assert(page->type() == ZPageTypeLarge, "Invalid page type"); + + // Undo page allocation + ZHeap::heap()->undo_alloc_page(page); + return true; +} + +bool ZObjectAllocator::undo_alloc_medium_object(ZPage* page, uintptr_t addr, size_t size) { + assert(page->type() == ZPageTypeMedium, "Invalid page type"); + + // Try atomic undo on shared page + return page->undo_alloc_object_atomic(addr, size); +} + +bool ZObjectAllocator::undo_alloc_small_object_from_nonworker(ZPage* page, uintptr_t addr, size_t size) { + assert(page->type() == ZPageTypeSmall, "Invalid page type"); + + if (ZThread::is_java()) { + // Try undo allocation in TLAB + if (Thread::current()->tlab().undo_allocate((HeapWord*)addr, ZUtils::bytes_to_words(size))) { + return true; + } + } + + // Try atomic undo on shared page + return page->undo_alloc_object_atomic(addr, size); +} + +bool ZObjectAllocator::undo_alloc_small_object_from_worker(ZPage* page, uintptr_t addr, size_t size) { + assert(page->type() == ZPageTypeSmall, "Invalid page type"); + assert(page == _worker_small_page.get(), "Invalid page"); + + // Non-atomic undo on worker-local page + const bool success = page->undo_alloc_object(addr, size); + assert(success, "Should always succeed"); + return success; +} + +bool ZObjectAllocator::undo_alloc_small_object(ZPage* page, uintptr_t addr, size_t size) { + if (ZThread::is_worker()) { + return undo_alloc_small_object_from_worker(page, addr, size); + } else { + return undo_alloc_small_object_from_nonworker(page, addr, size); + } +} + +bool ZObjectAllocator::undo_alloc_object(ZPage* page, uintptr_t addr, size_t size) { + const uint8_t type = page->type(); + + if (type == ZPageTypeSmall) { + return undo_alloc_small_object(page, addr, size); + } else if (type == ZPageTypeMedium) { + return undo_alloc_medium_object(page, addr, size); + } else { + return undo_alloc_large_object(page); + } +} + +void ZObjectAllocator::undo_alloc_object_for_relocation(ZPage* page, uintptr_t addr, size_t size) { + if (undo_alloc_object(page, addr, size)) { + ZStatInc(ZCounterUndoObjectAllocationSucceeded); + } else { + ZStatInc(ZCounterUndoObjectAllocationFailed); + log_trace(gc)("Failed to undo object allocation: " PTR_FORMAT ", Size: " SIZE_FORMAT ", Thread: " PTR_FORMAT " (%s)", + addr, size, ZThread::id(), ZThread::name()); + } +} + +size_t ZObjectAllocator::used() const { + size_t total_used = 0; + + ZPerCPUConstIterator iter(&_used); + for (const size_t* cpu_used; iter.next(&cpu_used);) { + total_used += *cpu_used; + } + + return total_used; +} + +size_t ZObjectAllocator::remaining() const { + assert(ZThread::is_java(), "Should be a Java thread"); + + ZPage* page = _shared_small_page.get(); + if (page != NULL) { + return page->remaining(); + } + + return 0; +} + +void ZObjectAllocator::retire_tlabs() { + ZStatTimer timer(ZSubPhasePauseRetireTLABS); + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + + // Retire TLABs + if (UseTLAB) { + ZCollectedHeap* heap = ZCollectedHeap::heap(); + heap->accumulate_statistics_all_tlabs(); + heap->ensure_parsability(true /* retire_tlabs */); + heap->resize_all_tlabs(); + } + + // Reset used + _used.set_all(0); + + // Reset allocation pages + _shared_medium_page.set(NULL); + _shared_small_page.set_all(NULL); + _worker_small_page.set_all(NULL); +} + +static void remap_tlab_address(HeapWord** p) { + *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p); +} + +void ZObjectAllocator::remap_tlabs() { + ZStatTimer timer(ZSubPhasePauseRemapTLABS); + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + + if (UseTLAB) { + for (JavaThreadIteratorWithHandle iter; JavaThread* thread = iter.next(); ) { + thread->tlab().addresses_do(remap_tlab_address); + } + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zObjectAllocator.hpp 2018-06-01 22:30:51.166336549 +0200 @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZOBJECTALLOCATOR_HPP +#define SHARE_GC_Z_ZOBJECTALLOCATOR_HPP + +#include "gc/z/zAllocationFlags.hpp" +#include "gc/z/zPage.hpp" +#include "gc/z/zValue.hpp" +#include "memory/allocation.hpp" + +class ZObjectAllocator { +private: + const uint _nworkers; + ZPerCPU _used; + ZContended _shared_medium_page; + ZPerCPU _shared_small_page; + ZPerWorker _worker_small_page; + + ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags); + + // Allocate an object in a shared page. Allocate and + // atomically install a new page if neccesary. + uintptr_t alloc_object_in_shared_page(ZPage** shared_page, + uint8_t page_type, + size_t page_size, + size_t size, + ZAllocationFlags flags); + + uintptr_t alloc_large_object(size_t size, ZAllocationFlags flags); + uintptr_t alloc_medium_object(size_t size, ZAllocationFlags flags); + uintptr_t alloc_small_object_from_nonworker(size_t size, ZAllocationFlags flags); + uintptr_t alloc_small_object_from_worker(size_t size, ZAllocationFlags flags); + uintptr_t alloc_small_object(size_t size, ZAllocationFlags flags); + uintptr_t alloc_object(size_t size, ZAllocationFlags flags); + + bool undo_alloc_large_object(ZPage* page); + bool undo_alloc_medium_object(ZPage* page, uintptr_t addr, size_t size); + bool undo_alloc_small_object_from_nonworker(ZPage* page, uintptr_t addr, size_t size); + bool undo_alloc_small_object_from_worker(ZPage* page, uintptr_t addr, size_t size); + bool undo_alloc_small_object(ZPage* page, uintptr_t addr, size_t size); + bool undo_alloc_object(ZPage* page, uintptr_t addr, size_t size); + +public: + ZObjectAllocator(uint nworkers); + + uintptr_t alloc_object(size_t size); + + uintptr_t alloc_object_for_relocation(size_t size); + void undo_alloc_object_for_relocation(ZPage* page, uintptr_t addr, size_t size); + + size_t used() const; + size_t remaining() const; + + void retire_tlabs(); + void remap_tlabs(); +}; + +#endif // SHARE_GC_Z_ZOBJECTALLOCATOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zOop.hpp 2018-06-01 22:30:51.539352650 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZOOP_HPP +#define SHARE_GC_Z_ZOOP_HPP + +#include "memory/allocation.hpp" +#include "oops/oopsHierarchy.hpp" + +class ZOop : public AllStatic { +public: + static oop to_oop(uintptr_t value); + static uintptr_t to_address(oop o); + + static bool is_good(oop o); + static bool is_good_or_null(oop o); + + static oop good(oop); +}; + +#endif // SHARE_GC_Z_ZOOP_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zOop.inline.hpp 2018-06-01 22:30:51.916368924 +0200 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZOOP_INLINE_HPP +#define SHARE_GC_Z_ZOOP_INLINE_HPP + +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zOop.hpp" +#include "oops/oopsHierarchy.hpp" + +inline oop ZOop::to_oop(uintptr_t value) { + return cast_to_oop(value); +} + +inline uintptr_t ZOop::to_address(oop o) { + return cast_from_oop(o); +} + +inline bool ZOop::is_good(oop o) { + return ZAddress::is_good(to_address(o)); +} + +inline bool ZOop::is_good_or_null(oop o) { + return ZAddress::is_good_or_null(to_address(o)); +} + +inline oop ZOop::good(oop o) { + return to_oop(ZAddress::good(to_address(o))); +} + +#endif // SHARE_GC_Z_ZOOP_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zOopClosures.cpp 2018-06-01 22:30:52.282384722 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zHeap.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zOop.inline.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/access.inline.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +static void z_verify_loaded_object(const oop* p, const oop obj) { + guarantee(ZOop::is_good_or_null(obj), + "Bad oop " PTR_FORMAT " found at " PTR_FORMAT ", expected " PTR_FORMAT, + p2i(obj), p2i(p), p2i(ZOop::good(obj))); + guarantee(oopDesc::is_oop_or_null(obj), + "Bad object " PTR_FORMAT " found at " PTR_FORMAT, + p2i(obj), p2i(p)); +} + +ZVerifyHeapOopClosure::ZVerifyHeapOopClosure(oop base) + : _base(base) {} + +void ZVerifyHeapOopClosure::do_oop(oop* p) { + guarantee(ZHeap::heap()->is_in((uintptr_t)p), "oop* " PTR_FORMAT " not in heap", p2i(p)); + + const oop obj = HeapAccess::oop_load_at(_base, _base->field_offset(p)); + z_verify_loaded_object(p, obj); +} + +void ZVerifyHeapOopClosure::do_oop(narrowOop* p) { + ShouldNotReachHere(); +} + +void ZVerifyRootOopClosure::do_oop(oop* p) { + guarantee(!ZHeap::heap()->is_in((uintptr_t)p), "oop* " PTR_FORMAT " in heap", p2i(p)); + + const oop obj = RootAccess<>::oop_load(p); + z_verify_loaded_object(p, obj); +} + +void ZVerifyRootOopClosure::do_oop(narrowOop* p) { + ShouldNotReachHere(); +} + +void ZVerifyObjectClosure::do_object(oop o) { + ZVerifyHeapOopClosure cl(o); + o->oop_iterate(&cl); +} + +// Generate Z specialized oop_oop_iterate functions. +SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_Z(ALL_KLASS_OOP_OOP_ITERATE_DEFN) --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zOopClosures.hpp 2018-06-01 22:30:52.656400867 +0200 @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZOOPCLOSURES_HPP +#define SHARE_GC_Z_ZOOPCLOSURES_HPP + +#include "memory/iterator.hpp" + +class ZLoadBarrierOopClosure : public ExtendedOopClosure { +public: + void do_oop_nv(oop* p); + void do_oop_nv(narrowOop* p); + + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + +#ifdef ASSERT + virtual bool should_verify_oops() { + return false; + } +#endif +}; + +class ZMarkRootOopClosure : public OopClosure { +public: + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class ZRelocateRootOopClosure : public OopClosure { +public: + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +template +class ZMarkBarrierOopClosure : public ExtendedOopClosure { +public: + ZMarkBarrierOopClosure(); + + void do_oop_nv(oop* p); + void do_oop_nv(narrowOop* p); + + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + +#ifdef ASSERT + virtual bool should_verify_oops() { + return false; + } +#endif +}; + +class ZPhantomIsAliveObjectClosure : public BoolObjectClosure { +public: + virtual bool do_object_b(oop o); +}; + +class ZPhantomKeepAliveOopClosure : public OopClosure { +public: + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class ZPhantomCleanOopClosure : public OopClosure { +public: + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class ZVerifyHeapOopClosure : public ExtendedOopClosure { +private: + const oop _base; + +public: + ZVerifyHeapOopClosure(oop base); + + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + +#ifdef ASSERT + // Verification handled by the closure itself. + virtual bool should_verify_oops() { + return false; + } +#endif +}; + +class ZVerifyRootOopClosure : public OopClosure { +public: + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class ZVerifyObjectClosure : public ObjectClosure { +public: + virtual void do_object(oop o); +}; + +#endif // SHARE_GC_Z_ZOOPCLOSURES_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zOopClosures.inline.hpp 2018-06-01 22:30:53.041417485 +0200 @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZOOPCLOSURES_INLINE_HPP +#define SHARE_GC_Z_ZOOPCLOSURES_INLINE_HPP + +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zOop.inline.hpp" +#include "gc/z/zOopClosures.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" + +inline void ZLoadBarrierOopClosure::do_oop_nv(oop* p) { + ZBarrier::load_barrier_on_oop_field(p); +} + +inline void ZLoadBarrierOopClosure::do_oop_nv(narrowOop* p) { + ShouldNotReachHere(); +} + +inline void ZLoadBarrierOopClosure::do_oop(oop* p) { + do_oop_nv(p); +} + +inline void ZLoadBarrierOopClosure::do_oop(narrowOop* p) { + do_oop_nv(p); +} + +inline void ZMarkRootOopClosure::do_oop(oop* p) { + ZBarrier::mark_barrier_on_root_oop_field(p); +} + +inline void ZMarkRootOopClosure::do_oop(narrowOop* p) { + ShouldNotReachHere(); +} + +inline void ZRelocateRootOopClosure::do_oop(oop* p) { + ZBarrier::relocate_barrier_on_root_oop_field(p); +} + +inline void ZRelocateRootOopClosure::do_oop(narrowOop* p) { + ShouldNotReachHere(); +} + +template +inline ZMarkBarrierOopClosure::ZMarkBarrierOopClosure() : + ExtendedOopClosure(finalizable ? NULL : ZHeap::heap()->reference_discoverer()) {} + +template +inline void ZMarkBarrierOopClosure::do_oop_nv(oop* p) { + ZBarrier::mark_barrier_on_oop_field(p, finalizable); +} + +template +inline void ZMarkBarrierOopClosure::do_oop_nv(narrowOop* p) { + ShouldNotReachHere(); +} + +template +inline void ZMarkBarrierOopClosure::do_oop(oop* p) { + do_oop_nv(p); +} + +template +inline void ZMarkBarrierOopClosure::do_oop(narrowOop* p) { + do_oop_nv(p); +} + +inline bool ZPhantomIsAliveObjectClosure::do_object_b(oop o) { + return ZBarrier::is_alive_barrier_on_phantom_oop(o); +} + +inline void ZPhantomKeepAliveOopClosure::do_oop(oop* p) { + ZBarrier::keep_alive_barrier_on_phantom_oop_field(p); +} + +inline void ZPhantomKeepAliveOopClosure::do_oop(narrowOop* p) { + ShouldNotReachHere(); +} + +inline void ZPhantomCleanOopClosure::do_oop(oop* p) { + // Read the oop once, to make sure the liveness check + // and the later clearing uses the same value. + const oop obj = *(volatile oop*)p; + if (ZBarrier::is_alive_barrier_on_phantom_oop(obj)) { + ZBarrier::keep_alive_barrier_on_phantom_oop_field(p); + } else { + // The destination could have been modified/reused, in which case + // we don't want to clear it. However, no one could write the same + // oop here again (the object would be strongly live and we would + // not consider clearing such oops), so therefore we don't have an + // ABA problem here. + Atomic::cmpxchg(oop(NULL), p, obj); + } +} + +inline void ZPhantomCleanOopClosure::do_oop(narrowOop* p) { + ShouldNotReachHere(); +} + +#endif // SHARE_GC_Z_ZOOPCLOSURES_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zOopClosures.specialized.hpp 2018-06-01 22:30:53.416433673 +0200 @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZSPECIALIZEDOOPCLOSURES_HPP +#define SHARE_GC_Z_ZSPECIALIZEDOOPCLOSURES_HPP + +class ZLoadBarrierOopClosure; +template class ZMarkBarrierOopClosure; + +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_Z(f) \ + f(ZLoadBarrierOopClosure,_nv) \ + f(ZMarkBarrierOopClosure,_nv) \ + f(ZMarkBarrierOopClosure,_nv) + +#endif // SHARE_GC_Z_ZSPECIALIZEDOOPCLOSURES_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPage.cpp 2018-06-01 22:30:53.810450680 +0200 @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zForwardingTable.inline.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zLiveMap.inline.hpp" +#include "gc/z/zMark.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zPhysicalMemory.inline.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zThread.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "logging/log.hpp" +#include "runtime/orderAccess.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond); + +ZPage::ZPage(uint8_t type, ZVirtualMemory vmem, ZPhysicalMemory pmem) : + _type(type), + _pinned(0), + _numa_id((uint8_t)-1), + _seqnum(0), + _virtual(vmem), + _top(start()), + _livemap(object_max_count()), + _refcount(0), + _forwarding(), + _physical(pmem) { + assert(!_physical.is_null(), "Should not be null"); + assert(!_virtual.is_null(), "Should not be null"); + assert((type == ZPageTypeSmall && size() == ZPageSizeSmall) || + (type == ZPageTypeMedium && size() == ZPageSizeMedium) || + (type == ZPageTypeLarge && is_aligned(size(), ZPageSizeMin)), + "Page type/size mismatch"); +} + +ZPage::~ZPage() { + assert(!is_active(), "Should not be active"); + assert(is_detached(), "Should be detached"); +} + +void ZPage::reset() { + assert(!is_active(), "Should not be active"); + assert(!is_pinned(), "Should not be pinned"); + assert(!is_detached(), "Should not be detached"); + + _seqnum = ZGlobalSeqNum; + _top = start(); + _livemap.reset(); + + // Make sure we don't make the page active before + // the reset of the above fields are visible. + OrderAccess::storestore(); + + _refcount = 1; +} + +uintptr_t ZPage::relocate_object_inner(uintptr_t from_index, uintptr_t from_offset) { + ZForwardingTableCursor cursor; + + // Lookup address in forwarding table + const ZForwardingTableEntry entry = _forwarding.find(from_index, &cursor); + if (entry.from_index() == from_index) { + // Already relocated, return new address + return entry.to_offset(); + } + + // Not found in forwarding table, relocate object + assert(is_object_marked(from_offset), "Should be marked"); + + if (is_pinned()) { + // In-place forward + return _forwarding.insert(from_index, from_offset, &cursor); + } + + // Allocate object + const uintptr_t from_good = ZAddress::good(from_offset); + const size_t size = ZUtils::object_size(from_good); + const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size); + if (to_good == 0) { + // Failed, in-place forward + return _forwarding.insert(from_index, from_offset, &cursor); + } + + // Copy object + ZUtils::object_copy(from_good, to_good, size); + + // Update forwarding table + const uintptr_t to_offset = ZAddress::offset(to_good); + const uintptr_t to_offset_final = _forwarding.insert(from_index, to_offset, &cursor); + if (to_offset_final == to_offset) { + // Relocation succeeded + return to_offset; + } + + // Relocation contention + ZStatInc(ZCounterRelocationContention); + log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT + ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT, + ZThread::id(), ZThread::name(), p2i(this), cursor, from_good, size); + + // Try undo allocation + ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size); + + return to_offset_final; +} + +uintptr_t ZPage::relocate_object(uintptr_t from) { + assert(ZHeap::heap()->is_relocating(from), "Should be relocating"); + + const uintptr_t from_offset = ZAddress::offset(from); + const uintptr_t from_index = (from_offset - start()) >> object_alignment_shift(); + const uintptr_t to_offset = relocate_object_inner(from_index, from_offset); + if (from_offset == to_offset) { + // In-place forwarding, pin page + set_pinned(); + } + + return ZAddress::good(to_offset); +} + +uintptr_t ZPage::forward_object(uintptr_t from) { + assert(ZHeap::heap()->is_relocating(from), "Should be relocated"); + + // Lookup address in forwarding table + const uintptr_t from_offset = ZAddress::offset(from); + const uintptr_t from_index = (from_offset - start()) >> object_alignment_shift(); + const ZForwardingTableEntry entry = _forwarding.find(from_index); + assert(entry.from_index() == from_index, "Should be forwarded"); + + return ZAddress::good(entry.to_offset()); +} + +void ZPage::print_on(outputStream* out) const { + out->print_cr(" %-6s " PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT " %s%s%s%s%s%s", + type_to_string(), start(), top(), end(), + is_allocating() ? " Allocating" : "", + is_relocatable() ? " Relocatable" : "", + is_forwarding() ? " Forwarding" : "", + is_pinned() ? " Pinned" : "", + is_detached() ? " Detached" : "", + !is_active() ? " Inactive" : ""); +} + +void ZPage::print() const { + print_on(tty); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPage.hpp 2018-06-01 22:30:54.198467428 +0200 @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGE_HPP +#define SHARE_GC_Z_ZPAGE_HPP + +#include "gc/z/zForwardingTable.hpp" +#include "gc/z/zList.hpp" +#include "gc/z/zLiveMap.hpp" +#include "gc/z/zPhysicalMemory.hpp" +#include "gc/z/zVirtualMemory.hpp" +#include "memory/allocation.hpp" + +class ZPage : public CHeapObj { + friend class VMStructs; + friend class ZList; + +private: + // Always hot + const uint8_t _type; // Page type + volatile uint8_t _pinned; // Pinned flag + uint8_t _numa_id; // NUMA node affinity + uint32_t _seqnum; // Allocation sequence number + const ZVirtualMemory _virtual; // Virtual start/end address + volatile uintptr_t _top; // Virtual top address + ZLiveMap _livemap; // Live map + + // Hot when relocated and cached + volatile uint32_t _refcount; // Page reference count + ZForwardingTable _forwarding; // Forwarding table + ZPhysicalMemory _physical; // Physical memory for page + ZListNode _node; // Page list node + + const char* type_to_string() const; + uint32_t object_max_count() const; + uintptr_t relocate_object_inner(uintptr_t from_index, uintptr_t from_offset); + + bool is_object_marked(uintptr_t addr) const; + bool is_object_strongly_marked(uintptr_t addr) const; + +public: + ZPage(uint8_t type, ZVirtualMemory vmem, ZPhysicalMemory pmem); + ~ZPage(); + + size_t object_alignment_shift() const; + size_t object_alignment() const; + + uint8_t type() const; + uintptr_t start() const; + uintptr_t end() const; + size_t size() const; + uintptr_t top() const; + size_t remaining() const; + + uint8_t numa_id(); + + ZPhysicalMemory& physical_memory(); + const ZVirtualMemory& virtual_memory() const; + + void reset(); + + bool inc_refcount(); + bool dec_refcount(); + + bool is_in(uintptr_t addr) const; + + uintptr_t block_start(uintptr_t addr) const; + size_t block_size(uintptr_t addr) const; + bool block_is_obj(uintptr_t addr) const; + + bool is_active() const; + bool is_allocating() const; + bool is_relocatable() const; + bool is_detached() const; + + bool is_mapped() const; + void set_pre_mapped(); + + bool is_pinned() const; + void set_pinned(); + + bool is_forwarding() const; + void set_forwarding(); + void reset_forwarding(); + void verify_forwarding() const; + + bool is_marked() const; + bool is_object_live(uintptr_t addr) const; + bool is_object_strongly_live(uintptr_t addr) const; + bool mark_object(uintptr_t addr, bool finalizable, bool& inc_live); + + void inc_live_atomic(uint32_t objects, size_t bytes); + size_t live_bytes() const; + + void object_iterate(ObjectClosure* cl); + + uintptr_t alloc_object(size_t size); + uintptr_t alloc_object_atomic(size_t size); + + bool undo_alloc_object(uintptr_t addr, size_t size); + bool undo_alloc_object_atomic(uintptr_t addr, size_t size); + + uintptr_t relocate_object(uintptr_t from); + uintptr_t forward_object(uintptr_t from); + + void print_on(outputStream* out) const; + void print() const; +}; + +#endif // SHARE_GC_Z_ZPAGE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPage.inline.hpp 2018-06-01 22:30:54.578483832 +0200 @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP +#define SHARE_GC_Z_ZPAGE_INLINE_HPP + +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zForwardingTable.inline.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zLiveMap.inline.hpp" +#include "gc/z/zMark.hpp" +#include "gc/z/zNUMA.hpp" +#include "gc/z/zPage.hpp" +#include "gc/z/zPhysicalMemory.inline.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "gc/z/zVirtualMemory.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +inline const char* ZPage::type_to_string() const { + switch (type()) { + case ZPageTypeSmall: + return "Small"; + + case ZPageTypeMedium: + return "Medium"; + + default: + assert(type() == ZPageTypeLarge, "Invalid page type"); + return "Large"; + } +} + +inline uint32_t ZPage::object_max_count() const { + switch (type()) { + case ZPageTypeLarge: + // A large page can only contain a single + // object aligned to the start of the page. + return 1; + + default: + return (uint32_t)(size() >> object_alignment_shift()); + } +} + +inline size_t ZPage::object_alignment_shift() const { + switch (type()) { + case ZPageTypeSmall: + return ZObjectAlignmentSmallShift; + + case ZPageTypeMedium: + return ZObjectAlignmentMediumShift; + + default: + assert(type() == ZPageTypeLarge, "Invalid page type"); + return ZObjectAlignmentLargeShift; + } +} + +inline size_t ZPage::object_alignment() const { + switch (type()) { + case ZPageTypeSmall: + return ZObjectAlignmentSmall; + + case ZPageTypeMedium: + return ZObjectAlignmentMedium; + + default: + assert(type() == ZPageTypeLarge, "Invalid page type"); + return ZObjectAlignmentLarge; + } +} + +inline uint8_t ZPage::type() const { + return _type; +} + +inline uintptr_t ZPage::start() const { + return _virtual.start(); +} + +inline uintptr_t ZPage::end() const { + return _virtual.end(); +} + +inline size_t ZPage::size() const { + return _virtual.size(); +} + +inline uintptr_t ZPage::top() const { + return _top; +} + +inline size_t ZPage::remaining() const { + return end() - top(); +} + +inline ZPhysicalMemory& ZPage::physical_memory() { + return _physical; +} + +inline const ZVirtualMemory& ZPage::virtual_memory() const { + return _virtual; +} + +inline uint8_t ZPage::numa_id() { + if (_numa_id == (uint8_t)-1) { + _numa_id = (uint8_t)ZNUMA::memory_id(ZAddress::good(start())); + } + + return _numa_id; +} + +inline bool ZPage::inc_refcount() { + for (uint32_t prev_refcount = _refcount; prev_refcount > 0; prev_refcount = _refcount) { + if (Atomic::cmpxchg(prev_refcount + 1, &_refcount, prev_refcount) == prev_refcount) { + return true; + } + } + return false; +} + +inline bool ZPage::dec_refcount() { + assert(is_active(), "Should be active"); + return Atomic::sub(1u, &_refcount) == 0; +} + +inline bool ZPage::is_in(uintptr_t addr) const { + const uintptr_t offset = ZAddress::offset(addr); + return offset >= start() && offset < top(); +} + +inline uintptr_t ZPage::block_start(uintptr_t addr) const { + if (block_is_obj(addr)) { + return addr; + } else { + return ZAddress::good(top()); + } +} + +inline size_t ZPage::block_size(uintptr_t addr) const { + if (block_is_obj(addr)) { + return ZUtils::object_size(addr); + } else { + return end() - top(); + } +} + +inline bool ZPage::block_is_obj(uintptr_t addr) const { + return ZAddress::offset(addr) < top(); +} + +inline bool ZPage::is_active() const { + return _refcount > 0; +} + +inline bool ZPage::is_allocating() const { + return is_active() && _seqnum == ZGlobalSeqNum; +} + +inline bool ZPage::is_relocatable() const { + return is_active() && _seqnum < ZGlobalSeqNum; +} + +inline bool ZPage::is_detached() const { + return _physical.is_null(); +} + +inline bool ZPage::is_mapped() const { + return _seqnum > 0; +} + +inline void ZPage::set_pre_mapped() { + // The _seqnum variable is also used to signal that the virtual and physical + // memory has been mapped. So, we need to set it to non-zero when the memory + // has been pre-mapped. + _seqnum = 1; +} + +inline bool ZPage::is_pinned() const { + return _pinned; +} + +inline void ZPage::set_pinned() { + _pinned = 1; +} + +inline bool ZPage::is_forwarding() const { + return !_forwarding.is_null(); +} + +inline void ZPage::set_forwarding() { + assert(is_marked(), "Should be marked"); + _forwarding.setup(_livemap.live_objects()); +} + +inline void ZPage::reset_forwarding() { + _forwarding.reset(); + _pinned = 0; +} + +inline void ZPage::verify_forwarding() const { + _forwarding.verify(object_max_count(), _livemap.live_objects()); +} + +inline bool ZPage::is_marked() const { + assert(is_relocatable(), "Invalid page state"); + return _livemap.is_marked(); +} + +inline bool ZPage::is_object_marked(uintptr_t addr) const { + const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2; + return _livemap.get(index); +} + +inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const { + const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2; + return _livemap.get(index + 1); +} + +inline bool ZPage::is_object_live(uintptr_t addr) const { + return is_allocating() || is_object_marked(addr); +} + +inline bool ZPage::is_object_strongly_live(uintptr_t addr) const { + return is_allocating() || is_object_strongly_marked(addr); +} + +inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) { + assert(ZAddress::is_marked(addr), "Invalid address"); + assert(is_relocatable(), "Invalid page state"); + assert(is_in(addr), "Invalid address"); + + // Set mark bit + const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2; + return _livemap.set_atomic(index, finalizable, inc_live); +} + +inline void ZPage::inc_live_atomic(uint32_t objects, size_t bytes) { + _livemap.inc_live_atomic(objects, bytes); +} + +inline size_t ZPage::live_bytes() const { + assert(is_marked(), "Should be marked"); + return _livemap.live_bytes(); +} + +inline void ZPage::object_iterate(ObjectClosure* cl) { + _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift()); +} + +inline uintptr_t ZPage::alloc_object(size_t size) { + assert(is_allocating(), "Invalid state"); + + const size_t aligned_size = align_up(size, object_alignment()); + const uintptr_t addr = top(); + const uintptr_t new_top = addr + aligned_size; + + if (new_top > end()) { + // Not enough space left + return 0; + } + + _top = new_top; + + // Fill alignment padding if needed + if (aligned_size != size) { + ZUtils::insert_filler_object(addr + size, aligned_size - size); + } + + return ZAddress::good(addr); +} + +inline uintptr_t ZPage::alloc_object_atomic(size_t size) { + assert(is_allocating(), "Invalid state"); + + const size_t aligned_size = align_up(size, object_alignment()); + uintptr_t addr = top(); + + for (;;) { + const uintptr_t new_top = addr + aligned_size; + if (new_top > end()) { + // Not enough space left + return 0; + } + + const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, addr); + if (prev_top == addr) { + // Fill alignment padding if needed + if (aligned_size != size) { + ZUtils::insert_filler_object(addr + size, aligned_size - size); + } + + // Success + return ZAddress::good(addr); + } + + // Retry + addr = prev_top; + } +} + +inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) { + assert(is_allocating(), "Invalid state"); + + const uintptr_t offset = ZAddress::offset(addr); + const size_t aligned_size = align_up(size, object_alignment()); + const uintptr_t old_top = top(); + const uintptr_t new_top = old_top - aligned_size; + + if (new_top != offset) { + // Failed to undo allocation, not the last allocated object + return false; + } + + _top = new_top; + + // Success + return true; +} + +inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) { + assert(is_allocating(), "Invalid state"); + + const uintptr_t offset = ZAddress::offset(addr); + const size_t aligned_size = align_up(size, object_alignment()); + uintptr_t old_top = top(); + + for (;;) { + const uintptr_t new_top = old_top - aligned_size; + if (new_top != offset) { + // Failed to undo allocation, not the last allocated object + return false; + } + + const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, old_top); + if (prev_top == old_top) { + // Success + return true; + } + + // Retry + old_top = prev_top; + } +} + +#endif // SHARE_GC_Z_ZPAGE_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageAllocator.cpp 2018-06-01 22:30:54.975500968 +0200 @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zFuture.inline.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zLock.inline.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zPageAllocator.hpp" +#include "gc/z/zPageCache.inline.hpp" +#include "gc/z/zPreMappedMemory.inline.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTracer.inline.hpp" +#include "runtime/init.hpp" + +static const ZStatCounter ZCounterAllocationRate("Memory", "Allocation Rate", ZStatUnitBytesPerSecond); +static const ZStatCriticalPhase ZCriticalPhaseAllocationStall("Allocation Stall"); + +class ZPageAllocRequest : public StackObj { + friend class ZList; + +private: + const uint8_t _type; + const size_t _size; + const ZAllocationFlags _flags; + const unsigned int _total_collections; + ZListNode _node; + ZFuture _result; + +public: + ZPageAllocRequest(uint8_t type, size_t size, ZAllocationFlags flags, unsigned int total_collections) : + _type(type), + _size(size), + _flags(flags), + _total_collections(total_collections) {} + + uint8_t type() const { + return _type; + } + + size_t size() const { + return _size; + } + + ZAllocationFlags flags() const { + return _flags; + } + + unsigned int total_collections() const { + return _total_collections; + } + + ZPage* wait() { + return _result.get(); + } + + void satisfy(ZPage* page) { + _result.set(page); + } +}; + +ZPage* const ZPageAllocator::gc_marker = (ZPage*)-1; + +ZPageAllocator::ZPageAllocator(size_t min_capacity, size_t max_capacity, size_t max_reserve) : + _virtual(), + _physical(max_capacity, ZPageSizeMin), + _cache(), + _pre_mapped(_virtual, _physical, min_capacity), + _max_reserve(max_reserve), + _used_high(0), + _used_low(0), + _used(0), + _allocated(0), + _reclaimed(0), + _queue(), + _detached() {} + +bool ZPageAllocator::is_initialized() const { + return _physical.is_initialized() && + _virtual.is_initialized() && + _pre_mapped.is_initialized(); +} + +size_t ZPageAllocator::max_capacity() const { + return _physical.max_capacity(); +} + +size_t ZPageAllocator::capacity() const { + return _physical.capacity(); +} + +size_t ZPageAllocator::max_reserve() const { + return _max_reserve; +} + +size_t ZPageAllocator::used_high() const { + return _used_high; +} + +size_t ZPageAllocator::used_low() const { + return _used_low; +} + +size_t ZPageAllocator::used() const { + return _used; +} + +size_t ZPageAllocator::allocated() const { + return _allocated; +} + +size_t ZPageAllocator::reclaimed() const { + return _reclaimed > 0 ? (size_t)_reclaimed : 0; +} + +void ZPageAllocator::reset_statistics() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + _allocated = 0; + _reclaimed = 0; + _used_high = _used_low = _used; +} + +void ZPageAllocator::increase_used(size_t size, bool relocation) { + if (relocation) { + // Allocating a page for the purpose of relocation has a + // negative contribution to the number of relcaimed bytes. + _reclaimed -= size; + } + _allocated += size; + _used += size; + if (_used > _used_high) { + _used_high = _used; + } +} + +void ZPageAllocator::decrease_used(size_t size, bool reclaimed) { + if (reclaimed) { + // Only pages explicitly released with the reclaimed flag set + // counts as reclaimed bytes. This flag is typically true when + // a worker releases a page after relocation, and is typically + // false when we release a page to undo an allocation. + _reclaimed += size; + } + _used -= size; + if (_used < _used_low) { + _used_low = _used; + } +} + +size_t ZPageAllocator::available(ZAllocationFlags flags) const { + size_t available = max_capacity() - used(); + assert(_physical.available() + _pre_mapped.available() + _cache.available() == available, "Should be equal"); + + if (flags.no_reserve()) { + // The memory reserve should not be considered free + available -= MIN2(available, max_reserve()); + } + + return available; +} + +ZPage* ZPageAllocator::create_page(uint8_t type, size_t size) { + // Allocate physical memory + const ZPhysicalMemory pmem = _physical.alloc(size); + if (pmem.is_null()) { + // Out of memory + return NULL; + } + + // Allocate virtual memory + const ZVirtualMemory vmem = _virtual.alloc(size); + if (vmem.is_null()) { + // Out of address space + _physical.free(pmem); + return NULL; + } + + // Allocate page + return new ZPage(type, vmem, pmem); +} + +void ZPageAllocator::flush_pre_mapped() { + if (_pre_mapped.available() == 0) { + return; + } + + // Detach the memory mapping. + detach_memory(_pre_mapped.virtual_memory(), _pre_mapped.physical_memory()); + + _pre_mapped.clear(); +} + +void ZPageAllocator::map_page(ZPage* page) { + // Map physical memory + _physical.map(page->physical_memory(), page->start()); +} + +void ZPageAllocator::detach_page(ZPage* page) { + // Detach the memory mapping. + detach_memory(page->virtual_memory(), page->physical_memory()); + + // Add to list of detached pages + _detached.insert_last(page); +} + +void ZPageAllocator::destroy_page(ZPage* page) { + assert(page->is_detached(), "Invalid page state"); + + // Free virtual memory + { + ZLocker locker(&_lock); + _virtual.free(page->virtual_memory()); + } + + delete page; +} + +void ZPageAllocator::flush_detached_pages(ZList* list) { + ZLocker locker(&_lock); + list->transfer(&_detached); +} + +void ZPageAllocator::flush_cache(size_t size) { + ZList list; + + _cache.flush(&list, size); + + for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) { + detach_page(page); + } +} + +void ZPageAllocator::check_out_of_memory_during_initialization() { + if (!is_init_completed()) { + vm_exit_during_initialization("java.lang.OutOfMemoryError", "Java heap too small"); + } +} + +ZPage* ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, ZAllocationFlags flags) { + const size_t available_total = available(flags); + if (available_total < size) { + // Not enough free memory + return NULL; + } + + // Try allocating from the page cache + ZPage* const cached_page = _cache.alloc_page(type, size); + if (cached_page != NULL) { + return cached_page; + } + + // Try allocate from the pre-mapped memory + ZPage* const pre_mapped_page = _pre_mapped.alloc_page(type, size); + if (pre_mapped_page != NULL) { + return pre_mapped_page; + } + + // Flush any remaining pre-mapped memory so that + // subsequent allocations can use the physical memory. + flush_pre_mapped(); + + // Check if physical memory is available + const size_t available_physical = _physical.available(); + if (available_physical < size) { + // Flush cache to free up more physical memory + flush_cache(size - available_physical); + } + + // Create new page and allocate physical memory + return create_page(type, size); +} + +ZPage* ZPageAllocator::alloc_page_common(uint8_t type, size_t size, ZAllocationFlags flags) { + ZPage* const page = alloc_page_common_inner(type, size, flags); + if (page == NULL) { + // Out of memory + return NULL; + } + + // Update used statistics + increase_used(size, flags.relocation()); + + // Send trace event + ZTracer::tracer()->report_page_alloc(size, used(), available(flags), _cache.available(), flags); + + return page; +} + +ZPage* ZPageAllocator::alloc_page_blocking(uint8_t type, size_t size, ZAllocationFlags flags) { + // Prepare to block + ZPageAllocRequest request(type, size, flags, ZCollectedHeap::heap()->total_collections()); + + _lock.lock(); + + // Try non-blocking allocation + ZPage* page = alloc_page_common(type, size, flags); + if (page == NULL) { + // Allocation failed, enqueue request + _queue.insert_last(&request); + } + + _lock.unlock(); + + if (page == NULL) { + // Allocation failed + ZStatTimer timer(ZCriticalPhaseAllocationStall); + + // We can only block if VM is fully initialized + check_out_of_memory_during_initialization(); + + do { + // Start asynchronous GC + ZCollectedHeap::heap()->collect(GCCause::_z_allocation_stall); + + // Wait for allocation to complete or fail + page = request.wait(); + } while (page == gc_marker); + } + + return page; +} + +ZPage* ZPageAllocator::alloc_page_nonblocking(uint8_t type, size_t size, ZAllocationFlags flags) { + ZLocker locker(&_lock); + return alloc_page_common(type, size, flags); +} + +ZPage* ZPageAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) { + ZPage* const page = flags.non_blocking() + ? alloc_page_nonblocking(type, size, flags) + : alloc_page_blocking(type, size, flags); + if (page == NULL) { + // Out of memory + return NULL; + } + + // Map page if needed + if (!page->is_mapped()) { + map_page(page); + } + + // Reset page. This updates the page's sequence number and must + // be done after page allocation, which potentially blocked in + // a safepoint where the global sequence number was updated. + page->reset(); + + // Update allocation statistics. Exclude worker threads to avoid + // artificial inflation of the allocation rate due to relocation. + if (!flags.worker_thread()) { + // Note that there are two allocation rate counters, which have + // different purposes and are sampled at different frequencies. + const size_t bytes = page->size(); + ZStatInc(ZCounterAllocationRate, bytes); + ZStatInc(ZStatAllocRate::counter(), bytes); + } + + return page; +} + +void ZPageAllocator::satisfy_alloc_queue() { + for (;;) { + ZPageAllocRequest* const request = _queue.first(); + if (request == NULL) { + // Allocation queue is empty + return; + } + + ZPage* const page = alloc_page_common(request->type(), request->size(), request->flags()); + if (page == NULL) { + // Allocation could not be satisfied, give up + return; + } + + // Allocation succeeded, dequeue and satisfy request. Note that + // the dequeue operation must happen first, since the request + // will immediately be deallocated once it has been satisfied. + _queue.remove(request); + request->satisfy(page); + } +} + +void ZPageAllocator::detach_memory(const ZVirtualMemory& vmem, ZPhysicalMemory& pmem) { + const uintptr_t addr = vmem.start(); + + // Unmap physical memory + _physical.unmap(pmem, addr); + + // Free physical memory + _physical.free(pmem); + + // Clear physical mapping + pmem.clear(); +} + +void ZPageAllocator::flip_page(ZPage* page) { + const ZPhysicalMemory& pmem = page->physical_memory(); + const uintptr_t addr = page->start(); + + // Flip physical mapping + _physical.flip(pmem, addr); +} + +void ZPageAllocator::flip_pre_mapped() { + if (_pre_mapped.available() == 0) { + // Nothing to flip + return; + } + + const ZPhysicalMemory& pmem = _pre_mapped.physical_memory(); + const ZVirtualMemory& vmem = _pre_mapped.virtual_memory(); + + // Flip physical mapping + _physical.flip(pmem, vmem.start()); +} + +void ZPageAllocator::free_page(ZPage* page, bool reclaimed) { + ZLocker locker(&_lock); + + // Update used statistics + decrease_used(page->size(), reclaimed); + + // Cache page + _cache.free_page(page); + + // Try satisfy blocked allocations + satisfy_alloc_queue(); +} + +void ZPageAllocator::check_out_of_memory() { + ZLocker locker(&_lock); + + ZPageAllocRequest* const first = _queue.first(); + if (first == NULL) { + // Allocation queue is empty + return; + } + + // Fail the allocation request if it was enqueued before the + // last GC cycle started, otherwise start a new GC cycle. + if (first->total_collections() < ZCollectedHeap::heap()->total_collections()) { + // Out of memory, fail all enqueued requests + for (ZPageAllocRequest* request = _queue.remove_first(); request != NULL; request = _queue.remove_first()) { + request->satisfy(NULL); + } + } else { + // Start another GC cycle, keep all enqueued requests + first->satisfy(gc_marker); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageAllocator.hpp 2018-06-01 22:30:55.350517156 +0200 @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGEALLOCATOR_HPP +#define SHARE_GC_Z_ZPAGEALLOCATOR_HPP + +#include "gc/z/zAllocationFlags.hpp" +#include "gc/z/zList.hpp" +#include "gc/z/zLock.hpp" +#include "gc/z/zPageCache.hpp" +#include "gc/z/zPhysicalMemory.hpp" +#include "gc/z/zPreMappedMemory.hpp" +#include "gc/z/zVirtualMemory.hpp" +#include "memory/allocation.hpp" + +class ZPageAllocRequest; + +class ZPageAllocator { + friend class VMStructs; + +private: + ZLock _lock; + ZVirtualMemoryManager _virtual; + ZPhysicalMemoryManager _physical; + ZPageCache _cache; + ZPreMappedMemory _pre_mapped; + const size_t _max_reserve; + size_t _used_high; + size_t _used_low; + size_t _used; + size_t _allocated; + ssize_t _reclaimed; + ZList _queue; + ZList _detached; + + static ZPage* const gc_marker; + + void increase_used(size_t size, bool relocation); + void decrease_used(size_t size, bool reclaimed); + + size_t available(ZAllocationFlags flags) const; + + ZPage* create_page(uint8_t type, size_t size); + void map_page(ZPage* page); + void detach_page(ZPage* page); + void flush_pre_mapped(); + void flush_cache(size_t size); + + void check_out_of_memory_during_initialization(); + + ZPage* alloc_page_common_inner(uint8_t type, size_t size, ZAllocationFlags flags); + ZPage* alloc_page_common(uint8_t type, size_t size, ZAllocationFlags flags); + ZPage* alloc_page_blocking(uint8_t type, size_t size, ZAllocationFlags flags); + ZPage* alloc_page_nonblocking(uint8_t type, size_t size, ZAllocationFlags flags); + + void satisfy_alloc_queue(); + + void detach_memory(const ZVirtualMemory& vmem, ZPhysicalMemory& pmem); + +public: + ZPageAllocator(size_t min_capacity, size_t max_capacity, size_t max_reserve); + + bool is_initialized() const; + + size_t max_capacity() const; + size_t capacity() const; + size_t max_reserve() const; + size_t used_high() const; + size_t used_low() const; + size_t used() const; + size_t allocated() const; + size_t reclaimed() const; + + void reset_statistics(); + + ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags); + void flip_page(ZPage* page); + void free_page(ZPage* page, bool reclaimed); + void destroy_page(ZPage* page); + + void flush_detached_pages(ZList* list); + + void flip_pre_mapped(); + + void check_out_of_memory(); +}; + +#endif // SHARE_GC_Z_ZPAGEALLOCATOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageCache.cpp 2018-06-01 22:30:55.728533472 +0200 @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zList.inline.hpp" +#include "gc/z/zNUMA.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zPageCache.hpp" +#include "gc/z/zStat.hpp" +#include "logging/log.hpp" + +static const ZStatCounter ZCounterPageCacheHitL1("Memory", "Page Cache Hit L1", ZStatUnitOpsPerSecond); +static const ZStatCounter ZCounterPageCacheHitL2("Memory", "Page Cache Hit L2", ZStatUnitOpsPerSecond); +static const ZStatCounter ZCounterPageCacheMiss("Memory", "Page Cache Miss", ZStatUnitOpsPerSecond); +static const ZStatCounter ZCounterPageCacheFlush("Memory", "Page Cache Flush", ZStatUnitBytesPerSecond); + +ZPageCache::ZPageCache() : + _available(0), + _small(), + _medium(), + _large() {} + +ZPage* ZPageCache::alloc_small_page() { + const uint32_t numa_id = ZNUMA::id(); + const uint32_t numa_count = ZNUMA::count(); + + // Try NUMA local page cache + ZPage* const l1_page = _small.get(numa_id).remove_first(); + if (l1_page != NULL) { + ZStatInc(ZCounterPageCacheHitL1); + return l1_page; + } + + // Try NUMA remote page cache(s) + uint32_t remote_numa_id = numa_id + 1; + const uint32_t remote_numa_count = numa_count - 1; + for (uint32_t i = 0; i < remote_numa_count; i++) { + if (remote_numa_id == numa_count) { + remote_numa_id = 0; + } + + ZPage* const l2_page = _small.get(remote_numa_id).remove_first(); + if (l2_page != NULL) { + ZStatInc(ZCounterPageCacheHitL2); + return l2_page; + } + + remote_numa_id++; + } + + ZStatInc(ZCounterPageCacheMiss); + return NULL; +} + +ZPage* ZPageCache::alloc_medium_page() { + ZPage* const l1_page = _medium.remove_first(); + if (l1_page != NULL) { + ZStatInc(ZCounterPageCacheHitL1); + return l1_page; + } + + ZStatInc(ZCounterPageCacheMiss); + return NULL; +} + +ZPage* ZPageCache::alloc_large_page(size_t size) { + // Find a page with the right size + ZListIterator iter(&_large); + for (ZPage* l1_page; iter.next(&l1_page);) { + if (l1_page->size() == size) { + // Page found + _large.remove(l1_page); + ZStatInc(ZCounterPageCacheHitL1); + return l1_page; + } + } + + ZStatInc(ZCounterPageCacheMiss); + return NULL; +} + +ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) { + ZPage* page; + + if (type == ZPageTypeSmall) { + page = alloc_small_page(); + } else if (type == ZPageTypeMedium) { + page = alloc_medium_page(); + } else { + page = alloc_large_page(size); + } + + if (page != NULL) { + _available -= page->size(); + } + + return page; +} + +void ZPageCache::free_page(ZPage* page) { + assert(!page->is_active(), "Invalid page state"); + assert(!page->is_pinned(), "Invalid page state"); + assert(!page->is_detached(), "Invalid page state"); + + const uint8_t type = page->type(); + if (type == ZPageTypeSmall) { + _small.get(page->numa_id()).insert_first(page); + } else if (type == ZPageTypeMedium) { + _medium.insert_first(page); + } else { + _large.insert_first(page); + } + + _available += page->size(); +} + +void ZPageCache::flush_list(ZList* from, size_t requested, ZList* to, size_t* flushed) { + while (*flushed < requested) { + // Flush least recently used + ZPage* const page = from->remove_last(); + if (page == NULL) { + break; + } + + *flushed += page->size(); + to->insert_last(page); + } +} + +void ZPageCache::flush_per_numa_lists(ZPerNUMA >* from, size_t requested, ZList* to, size_t* flushed) { + const uint32_t numa_count = ZNUMA::count(); + uint32_t numa_empty = 0; + uint32_t numa_next = 0; + + // Flush lists round-robin + while (*flushed < requested) { + ZPage* const page = from->get(numa_next).remove_last(); + + if (++numa_next == numa_count) { + numa_next = 0; + } + + if (page == NULL) { + // List is empty + if (++numa_empty == numa_count) { + // All lists are empty + break; + } + + // Try next list + continue; + } + + // Flush page + numa_empty = 0; + *flushed += page->size(); + to->insert_last(page); + } +} + +void ZPageCache::flush(ZList* to, size_t requested) { + size_t flushed = 0; + + // Prefer flushing large, then medium and last small pages + flush_list(&_large, requested, to, &flushed); + flush_list(&_medium, requested, to, &flushed); + flush_per_numa_lists(&_small, requested, to, &flushed); + + ZStatInc(ZCounterPageCacheFlush, flushed); + + log_info(gc, heap)("Page Cache Flushed: " + SIZE_FORMAT "M requested, " + SIZE_FORMAT "M(" SIZE_FORMAT "M->" SIZE_FORMAT "M) flushed", + requested / M, flushed / M , _available / M, (_available - flushed) / M); + + _available -= flushed; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageCache.hpp 2018-06-01 22:30:56.118550307 +0200 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGECACHE_HPP +#define SHARE_GC_Z_ZPAGECACHE_HPP + +#include "gc/z/zList.hpp" +#include "gc/z/zPage.hpp" +#include "gc/z/zValue.hpp" +#include "memory/allocation.hpp" + +class ZPageCache { +private: + size_t _available; + + ZPerNUMA > _small; + ZList _medium; + ZList _large; + + ZPage* alloc_small_page(); + ZPage* alloc_medium_page(); + ZPage* alloc_large_page(size_t size); + + void flush_list(ZList* from, size_t requested, ZList* to, size_t* flushed); + void flush_per_numa_lists(ZPerNUMA >* from, size_t requested, ZList* to, size_t* flushed); + +public: + ZPageCache(); + + size_t available() const; + + ZPage* alloc_page(uint8_t type, size_t size); + void free_page(ZPage* page); + + void flush(ZList* to, size_t requested); +}; + +#endif // SHARE_GC_Z_ZPAGECACHE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageCache.inline.hpp 2018-06-01 22:30:56.507567099 +0200 @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGECACHE_INLINE_HPP +#define SHARE_GC_Z_ZPAGECACHE_INLINE_HPP + +#include "gc/z/zPageCache.hpp" + +inline size_t ZPageCache::available() const { + return _available; +} + +#endif // SHARE_GC_Z_ZPAGECACHE_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageTable.cpp 2018-06-01 22:30:56.894583804 +0200 @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zPageTable.inline.hpp" +#include "runtime/orderAccess.hpp" +#include "utilities/debug.hpp" + +ZPageTable::ZPageTable() : + _map() {} + +ZPageTableEntry ZPageTable::get_entry(ZPage* page) const { + const uintptr_t addr = ZAddress::good(page->start()); + return _map.get(addr); +} + +void ZPageTable::put_entry(ZPage* page, ZPageTableEntry entry) { + // Make sure a newly created page is globally visible before + // updating the pagetable. + OrderAccess::storestore(); + + const uintptr_t start = ZAddress::good(page->start()); + const uintptr_t end = start + page->size(); + for (uintptr_t addr = start; addr < end; addr += ZPageSizeMin) { + _map.put(addr, entry); + } +} + +void ZPageTable::insert(ZPage* page) { + assert(get_entry(page).page() == NULL || + get_entry(page).page() == page, "Invalid entry"); + + // Cached pages stays in the pagetable and we must not re-insert + // those when they get re-allocated because they might also be + // relocating and we don't want to clear their relocating bit. + if (get_entry(page).page() == NULL) { + ZPageTableEntry entry(page, false /* relocating */); + put_entry(page, entry); + } + + assert(get_entry(page).page() == page, "Invalid entry"); +} + +void ZPageTable::remove(ZPage* page) { + assert(get_entry(page).page() == page, "Invalid entry"); + + ZPageTableEntry entry; + put_entry(page, entry); + + assert(get_entry(page).page() == NULL, "Invalid entry"); +} + +void ZPageTable::set_relocating(ZPage* page) { + assert(get_entry(page).page() == page, "Invalid entry"); + assert(!get_entry(page).relocating(), "Invalid entry"); + + ZPageTableEntry entry(page, true /* relocating */); + put_entry(page, entry); + + assert(get_entry(page).page() == page, "Invalid entry"); + assert(get_entry(page).relocating(), "Invalid entry"); +} + +void ZPageTable::clear_relocating(ZPage* page) { + assert(get_entry(page).page() == page, "Invalid entry"); + assert(get_entry(page).relocating(), "Invalid entry"); + + ZPageTableEntry entry(page, false /* relocating */); + put_entry(page, entry); + + assert(get_entry(page).page() == page, "Invalid entry"); + assert(!get_entry(page).relocating(), "Invalid entry"); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageTable.hpp 2018-06-01 22:30:57.267599905 +0200 @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGETABLE_HPP +#define SHARE_GC_Z_ZPAGETABLE_HPP + +#include "gc/z/zAddressRangeMap.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zPageTableEntry.hpp" +#include "memory/allocation.hpp" + +class ZPage; + +class ZPageTable { + friend class VMStructs; + friend class ZPageTableIterator; + +private: + ZAddressRangeMap _map; + + ZPageTableEntry get_entry(ZPage* page) const; + void put_entry(ZPage* page, ZPageTableEntry entry); + +public: + ZPageTable(); + + ZPage* get(uintptr_t addr) const; + void insert(ZPage* page); + void remove(ZPage* page); + + bool is_relocating(uintptr_t addr) const; + void set_relocating(ZPage* page); + void clear_relocating(ZPage* page); +}; + +class ZPageTableIterator : public StackObj { +private: + ZAddressRangeMapIterator _iter; + ZPage* _prev; + +public: + ZPageTableIterator(const ZPageTable* pagetable); + + bool next(ZPage** page); +}; + +#endif // SHARE_GC_Z_ZPAGETABLE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageTable.inline.hpp 2018-06-01 22:30:57.643616135 +0200 @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGETABLE_INLINE_HPP +#define SHARE_GC_Z_ZPAGETABLE_INLINE_HPP + +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zAddressRangeMap.inline.hpp" +#include "gc/z/zPageTable.hpp" + +inline ZPage* ZPageTable::get(uintptr_t addr) const { + return _map.get(addr).page(); +} + +inline bool ZPageTable::is_relocating(uintptr_t addr) const { + return _map.get(addr).relocating(); +} + +inline ZPageTableIterator::ZPageTableIterator(const ZPageTable* pagetable) : + _iter(&pagetable->_map), + _prev(NULL) {} + +inline bool ZPageTableIterator::next(ZPage** page) { + ZPageTableEntry entry; + + while (_iter.next(&entry)) { + ZPage* const next = entry.page(); + if (next != NULL && next != _prev) { + // Next page found + *page = _prev = next; + return true; + } + } + + // No more pages + return false; +} + +#endif // SHARE_GC_Z_ZPAGETABLE_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPageTableEntry.hpp 2018-06-01 22:30:58.028632754 +0200 @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPAGETABLEENTRY_HPP +#define SHARE_GC_Z_ZPAGETABLEENTRY_HPP + +#include "gc/z/zBitField.hpp" +#include "memory/allocation.hpp" + +// +// Page table entry layout +// ----------------------- +// +// 6 +// 3 1 0 +// +----------------------------------------------------------------------+-+ +// |11111111 11111111 11111111 11111111 11111111 11111111 11111111 1111111|1| +// +----------------------------------------------------------------------+-+ +// | | +// | 0-0 Relocating Flag (1-bit) * +// | +// | +// | +// * 63-1 Page address (63-bits) +// + +class ZPage; + +class ZPageTableEntry { +private: + typedef ZBitField field_relocating; + typedef ZBitField field_page; + + uint64_t _entry; + +public: + ZPageTableEntry() : + _entry(0) {} + + ZPageTableEntry(ZPage* page, bool relocating) : + _entry(field_page::encode(page) | + field_relocating::encode(relocating)) {} + + bool relocating() const { + return field_relocating::decode(_entry); + } + + ZPage* page() const { + return field_page::decode(_entry); + } +}; + +#endif // SHARE_GC_Z_ZPAGETABLEENTRY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPhysicalMemory.cpp 2018-06-01 22:30:58.410649244 +0200 @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zPhysicalMemory.inline.hpp" +#include "logging/log.hpp" +#include "memory/allocation.inline.hpp" +#include "services/memTracker.hpp" +#include "utilities/debug.hpp" + +ZPhysicalMemory::ZPhysicalMemory() : + _nsegments(0), + _segments(NULL) {} + +ZPhysicalMemory::ZPhysicalMemory(size_t size) : + _nsegments(0), + _segments(NULL) { + add_segment(ZPhysicalMemorySegment(0, size)); +} + +ZPhysicalMemory::ZPhysicalMemory(const ZPhysicalMemorySegment& segment) : + _nsegments(0), + _segments(NULL) { + add_segment(segment); +} + +size_t ZPhysicalMemory::size() const { + size_t size = 0; + + for (size_t i = 0; i < _nsegments; i++) { + size += _segments[i].size(); + } + + return size; +} + +void ZPhysicalMemory::add_segment(ZPhysicalMemorySegment segment) { + // Try merge with last segment + if (_nsegments > 0) { + ZPhysicalMemorySegment& last = _segments[_nsegments - 1]; + assert(last.end() <= segment.start(), "Segments added out of order"); + if (last.end() == segment.start()) { + // Merge + last.expand(segment.size()); + return; + } + } + + // Make room for a new segment + const size_t size = sizeof(ZPhysicalMemorySegment) * (_nsegments + 1); + _segments = (ZPhysicalMemorySegment*)ReallocateHeap((char*)_segments, size, mtGC); + + // Add new segment + _segments[_nsegments] = segment; + _nsegments++; +} + +ZPhysicalMemory ZPhysicalMemory::split(size_t split_size) { + // Only splitting of single-segment instances have been implemented. + assert(nsegments() == 1, "Can only have one segment"); + assert(split_size <= size(), "Invalid size"); + return ZPhysicalMemory(_segments[0].split(split_size)); +} + +void ZPhysicalMemory::clear() { + if (_segments != NULL) { + FreeHeap(_segments); + _segments = NULL; + _nsegments = 0; + } +} + +ZPhysicalMemoryManager::ZPhysicalMemoryManager(size_t max_capacity, size_t granule_size) : + _backing(max_capacity, granule_size), + _max_capacity(max_capacity), + _capacity(0), + _used(0) {} + +bool ZPhysicalMemoryManager::is_initialized() const { + return _backing.is_initialized(); +} + +bool ZPhysicalMemoryManager::ensure_available(size_t size) { + const size_t unused_capacity = _capacity - _used; + if (unused_capacity >= size) { + // Enough unused capacity available + return true; + } + + const size_t expand_with = size - unused_capacity; + const size_t new_capacity = _capacity + expand_with; + if (new_capacity > _max_capacity) { + // Can not expand beyond max capacity + return false; + } + + // Expand + if (!_backing.expand(_capacity, new_capacity)) { + log_error(gc)("Failed to expand Java heap with " SIZE_FORMAT "%s", + byte_size_in_proper_unit(expand_with), + proper_unit_for_byte_size(expand_with)); + return false; + } + + _capacity = new_capacity; + + return true; +} + +void ZPhysicalMemoryManager::nmt_commit(ZPhysicalMemory pmem, uintptr_t offset) { + const uintptr_t addr = _backing.nmt_address(offset); + const size_t size = pmem.size(); + MemTracker::record_virtual_memory_commit((void*)addr, size, CALLER_PC); +} + +void ZPhysicalMemoryManager::nmt_uncommit(ZPhysicalMemory pmem, uintptr_t offset) { + if (MemTracker::tracking_level() > NMT_minimal) { + const uintptr_t addr = _backing.nmt_address(offset); + const size_t size = pmem.size(); + + Tracker tracker(Tracker::uncommit); + tracker.record((address)addr, size); + } +} + +ZPhysicalMemory ZPhysicalMemoryManager::alloc(size_t size) { + if (!ensure_available(size)) { + // Not enough memory available + return ZPhysicalMemory(); + } + + _used += size; + return _backing.alloc(size); +} + +void ZPhysicalMemoryManager::free(ZPhysicalMemory pmem) { + _backing.free(pmem); + _used -= pmem.size(); +} + +void ZPhysicalMemoryManager::map(ZPhysicalMemory pmem, uintptr_t offset) { + // Map page + _backing.map(pmem, offset); + + // Update native memory tracker + nmt_commit(pmem, offset); +} + +void ZPhysicalMemoryManager::unmap(ZPhysicalMemory pmem, uintptr_t offset) { + // Update native memory tracker + nmt_uncommit(pmem, offset); + + // Unmap page + _backing.unmap(pmem, offset); +} + +void ZPhysicalMemoryManager::flip(ZPhysicalMemory pmem, uintptr_t offset) { + _backing.flip(pmem, offset); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPhysicalMemory.hpp 2018-06-01 22:30:58.800666078 +0200 @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPHYSICALMEMORY_HPP +#define SHARE_GC_Z_ZPHYSICALMEMORY_HPP + +#include "memory/allocation.hpp" +#include OS_CPU_HEADER(gc/z/zPhysicalMemoryBacking) + +class ZPhysicalMemorySegment { +private: + uintptr_t _start; + uintptr_t _end; + +public: + ZPhysicalMemorySegment(uintptr_t start, size_t size); + + uintptr_t start() const; + uintptr_t end() const; + size_t size() const; + + void expand(size_t size); + ZPhysicalMemorySegment split(size_t size); +}; + +class ZPhysicalMemory { +private: + size_t _nsegments; + ZPhysicalMemorySegment* _segments; + +public: + ZPhysicalMemory(); + ZPhysicalMemory(size_t size); + ZPhysicalMemory(const ZPhysicalMemorySegment& segment); + + bool is_null() const; + size_t size() const; + + size_t nsegments() const; + ZPhysicalMemorySegment segment(size_t index) const; + void add_segment(ZPhysicalMemorySegment segment); + + ZPhysicalMemory split(size_t size); + void clear(); +}; + +class ZPhysicalMemoryManager { + friend class VMStructs; + +private: + ZPhysicalMemoryBacking _backing; + const size_t _max_capacity; + size_t _capacity; + size_t _used; + + bool ensure_available(size_t size); + + void nmt_commit(ZPhysicalMemory pmem, uintptr_t offset); + void nmt_uncommit(ZPhysicalMemory pmem, uintptr_t offset); + +public: + ZPhysicalMemoryManager(size_t max_capacity, size_t granule_size); + + bool is_initialized() const; + + size_t max_capacity() const; + size_t capacity() const; + size_t used() const; + size_t available() const; + + ZPhysicalMemory alloc(size_t size); + void free(ZPhysicalMemory pmem); + + void map(ZPhysicalMemory pmem, uintptr_t offset); + void unmap(ZPhysicalMemory pmem, uintptr_t offset); + void flip(ZPhysicalMemory pmem, uintptr_t offset); +}; + +#endif // SHARE_GC_Z_ZPHYSICALMEMORY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp 2018-06-01 22:30:59.170682050 +0200 @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPHYSICALMEMORY_INLINE_HPP +#define SHARE_GC_Z_ZPHYSICALMEMORY_INLINE_HPP + +#include "gc/z/zPhysicalMemory.hpp" +#include "utilities/debug.hpp" + +inline ZPhysicalMemorySegment::ZPhysicalMemorySegment(uintptr_t start, size_t size) : + _start(start), + _end(start + size) {} + +inline uintptr_t ZPhysicalMemorySegment::start() const { + return _start; +} + +inline uintptr_t ZPhysicalMemorySegment::end() const { + return _end; +} + +inline size_t ZPhysicalMemorySegment::size() const { + return end() - start(); +} + +inline void ZPhysicalMemorySegment::expand(size_t size) { + _end += size; +} + +inline ZPhysicalMemorySegment ZPhysicalMemorySegment::split(size_t split_size) { + assert(split_size <= size(), "Invalid size"); + ZPhysicalMemorySegment segment(_start, split_size); + _start += split_size; + return segment; +} + +inline bool ZPhysicalMemory::is_null() const { + return _nsegments == 0; +} + +inline size_t ZPhysicalMemory::nsegments() const { + return _nsegments; +} + +inline ZPhysicalMemorySegment ZPhysicalMemory::segment(size_t index) const { + assert(index < _nsegments, "Invalid segment index"); + return _segments[index]; +} + +inline size_t ZPhysicalMemoryManager::max_capacity() const { + return _max_capacity; +} + +inline size_t ZPhysicalMemoryManager::capacity() const { + return _capacity; +} + +inline size_t ZPhysicalMemoryManager::used() const { + return _used; +} + +inline size_t ZPhysicalMemoryManager::available() const { + return _max_capacity - _used; +} + +#endif // SHARE_GC_Z_ZPHYSICALMEMORY_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPreMappedMemory.cpp 2018-06-01 22:30:59.543698151 +0200 @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zPhysicalMemory.inline.hpp" +#include "gc/z/zPreMappedMemory.inline.hpp" +#include "gc/z/zVirtualMemory.inline.hpp" +#include "logging/log.hpp" + +ZPreMappedMemory::ZPreMappedMemory(ZVirtualMemoryManager &vmm, ZPhysicalMemoryManager &pmm, size_t size) : + _vmem(), + _pmem(), + _initialized(false) { + if (!vmm.is_initialized() || !pmm.is_initialized()) { + // Not initialized + return; + } + + // Pre-mapping and pre-touching memory can take a long time. Log a message + // to help the user understand why the JVM might seem slow to start. + log_info(gc, init)("Pre-touching: %s", AlwaysPreTouch ? "Enabled" : "Disabled"); + log_info(gc, init)("Pre-mapping: " SIZE_FORMAT "M", size / M); + + _pmem = pmm.alloc(size); + if (_pmem.is_null()) { + // Out of memory + return; + } + + _vmem = vmm.alloc(size, true /* alloc_from_front */); + if (_vmem.is_null()) { + // Out of address space + pmm.free(_pmem); + return; + } + + // Map physical memory + pmm.map(_pmem, _vmem.start()); + + _initialized = true; +} + +ZPage* ZPreMappedMemory::alloc_page(uint8_t type, size_t size) { + if (size > available()) { + // Not enough pre-mapped memory + return NULL; + } + + // Take a chunk of the pre-mapped memory + const ZPhysicalMemory pmem = _pmem.split(size); + const ZVirtualMemory vmem = _vmem.split(size); + + ZPage* const page = new ZPage(type, vmem, pmem); + page->set_pre_mapped(); + + return page; +} + +void ZPreMappedMemory::clear() { + assert(_pmem.is_null(), "Should be detached"); + _vmem.clear(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPreMappedMemory.hpp 2018-06-01 22:30:59.929714813 +0200 @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPREMAPPEDMEMORY_HPP +#define SHARE_GC_Z_ZPREMAPPEDMEMORY_HPP + +#include "gc/z/zPhysicalMemory.hpp" +#include "gc/z/zVirtualMemory.hpp" +#include "memory/allocation.hpp" + +class ZPage; + +class ZPreMappedMemory { +private: + ZVirtualMemory _vmem; + ZPhysicalMemory _pmem; + bool _initialized; + +public: + ZPreMappedMemory(ZVirtualMemoryManager &vmm, ZPhysicalMemoryManager &pmm, size_t size); + + bool is_initialized() const; + + ZPhysicalMemory& physical_memory(); + const ZVirtualMemory& virtual_memory() const; + + size_t available() const; + + ZPage* alloc_page(uint8_t type, size_t size); + + void clear(); +}; + +#endif // SHARE_GC_Z_ZPREMAPPEDMEMORY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zPreMappedMemory.inline.hpp 2018-06-01 22:31:00.320731691 +0200 @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZPREMAPPEDMEMORY_INLINE_HPP +#define SHARE_GC_Z_ZPREMAPPEDMEMORY_INLINE_HPP + +#include "gc/z/zPreMappedMemory.hpp" + +inline bool ZPreMappedMemory::is_initialized() const { + return _initialized; +} + +inline ZPhysicalMemory& ZPreMappedMemory::physical_memory() { + return _pmem; +} + +inline const ZVirtualMemory& ZPreMappedMemory::virtual_memory() const { + return _vmem; +} + +inline size_t ZPreMappedMemory::available() const { + return _vmem.size(); +} + +#endif // SHARE_GC_Z_ZPREMAPPEDMEMORY_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zReferenceProcessor.cpp 2018-06-01 22:31:00.714748698 +0200 @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.inline.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/referenceProcessorStats.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zReferenceProcessor.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zTracer.inline.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "memory/universe.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/os.hpp" + +static const ZStatSubPhase ZSubPhaseConcurrentReferencesProcess("Concurrent References Process"); +static const ZStatSubPhase ZSubPhaseConcurrentReferencesEnqueue("Concurrent References Enqueue"); + +ZReferenceProcessor::ZReferenceProcessor(ZWorkers* workers) : + _workers(workers), + _soft_reference_policy(NULL), + _encountered_count(), + _discovered_count(), + _enqueued_count(), + _discovered_list(NULL), + _pending_list(NULL), + _pending_list_tail(_pending_list.addr()) {} + +void ZReferenceProcessor::set_soft_reference_policy(bool clear) { + static AlwaysClearPolicy always_clear_policy; + static LRUMaxHeapPolicy lru_max_heap_policy; + + if (clear) { + log_info(gc, ref)("Clearing All Soft References"); + _soft_reference_policy = &always_clear_policy; + } else { + _soft_reference_policy = &lru_max_heap_policy; + } + + _soft_reference_policy->setup(); +} + +void ZReferenceProcessor::update_soft_reference_clock() const { + const jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + java_lang_ref_SoftReference::set_clock(now); +} + +bool ZReferenceProcessor::is_reference_inactive(oop obj) const { + // A non-null next field means the reference is inactive + return java_lang_ref_Reference::next(obj) != NULL; +} + +ReferenceType ZReferenceProcessor::reference_type(oop obj) const { + return InstanceKlass::cast(obj->klass())->reference_type(); +} + +const char* ZReferenceProcessor::reference_type_name(ReferenceType type) const { + switch (type) { + case REF_SOFT: + return "Soft"; + + case REF_WEAK: + return "Weak"; + + case REF_FINAL: + return "Final"; + + case REF_PHANTOM: + return "Phantom"; + + default: + ShouldNotReachHere(); + return NULL; + } +} + +volatile oop* ZReferenceProcessor::reference_referent_addr(oop obj) const { + return (volatile oop*)java_lang_ref_Reference::referent_addr_raw(obj); +} + +oop ZReferenceProcessor::reference_referent(oop obj) const { + return *reference_referent_addr(obj); +} + +bool ZReferenceProcessor::is_referent_alive_or_null(oop obj, ReferenceType type) const { + volatile oop* const p = reference_referent_addr(obj); + + // Check if the referent is alive or null, in which case we don't want to discover + // the reference. It can only be null if the application called Reference.enqueue() + // or Reference.clear(). + if (type == REF_PHANTOM) { + const oop o = ZBarrier::weak_load_barrier_on_phantom_oop_field(p); + return o == NULL || ZHeap::heap()->is_object_live(ZOop::to_address(o)); + } else { + const oop o = ZBarrier::weak_load_barrier_on_weak_oop_field(p); + return o == NULL || ZHeap::heap()->is_object_strongly_live(ZOop::to_address(o)); + } +} + +bool ZReferenceProcessor::is_referent_softly_alive(oop obj, ReferenceType type) const { + if (type != REF_SOFT) { + // Not a soft reference + return false; + } + + // Ask soft reference policy + const jlong clock = java_lang_ref_SoftReference::clock(); + assert(clock != 0, "Clock not initialized"); + assert(_soft_reference_policy != NULL, "Policy not initialized"); + return !_soft_reference_policy->should_clear_reference(obj, clock); +} + +bool ZReferenceProcessor::should_drop_reference(oop obj, ReferenceType type) const { + // This check is racing with a call to Reference.clear() from the application. + // If the application clears the reference after this check it will still end + // up on the pending list, and there's nothing we can do about that without + // changing the Reference.clear() API. This check is also racing with a call + // to Reference.enqueue() from the application, which is unproblematic, since + // the application wants the reference to be enqueued anyway. + const oop o = reference_referent(obj); + if (o == NULL) { + // Reference has been cleared, by a call to Reference.enqueue() + // or Reference.clear() from the application, which means we + // should drop the reference. + return true; + } + + // Check if the referent is still alive, in which case we should + // drop the reference. + if (type == REF_PHANTOM) { + return ZBarrier::is_alive_barrier_on_phantom_oop(o); + } else { + return ZBarrier::is_alive_barrier_on_weak_oop(o); + } +} + +bool ZReferenceProcessor::should_mark_referent(ReferenceType type) const { + // Referents of final references (and its reachable sub graph) are + // always marked finalizable during discovery. This avoids the problem + // of later having to mark those objects if the referent is still final + // reachable during processing. + return type == REF_FINAL; +} + +bool ZReferenceProcessor::should_clear_referent(ReferenceType type) const { + // Referents that were not marked must be cleared + return !should_mark_referent(type); +} + +void ZReferenceProcessor::keep_referent_alive(oop obj, ReferenceType type) const { + volatile oop* const p = reference_referent_addr(obj); + if (type == REF_PHANTOM) { + ZBarrier::keep_alive_barrier_on_phantom_oop_field(p); + } else { + ZBarrier::keep_alive_barrier_on_weak_oop_field(p); + } +} + +bool ZReferenceProcessor::discover_reference(oop obj, ReferenceType type) { + if (!RegisterReferences) { + // Reference processing disabled + return false; + } + + log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s)", p2i(obj), reference_type_name(type)); + + // Update statistics + _encountered_count.get()[type]++; + + if (is_reference_inactive(obj) || + is_referent_alive_or_null(obj, type) || + is_referent_softly_alive(obj, type)) { + // Not discovered + return false; + } + + discover(obj, type); + + // Discovered + return true; +} + +void ZReferenceProcessor::discover(oop obj, ReferenceType type) { + log_trace(gc, ref)("Discovered Reference: " PTR_FORMAT " (%s)", p2i(obj), reference_type_name(type)); + + // Update statistics + _discovered_count.get()[type]++; + + // Mark referent finalizable + if (should_mark_referent(type)) { + oop* const referent_addr = (oop*)java_lang_ref_Reference::referent_addr_raw(obj); + ZBarrier::mark_barrier_on_oop_field(referent_addr, true /* finalizable */); + } + + // Add reference to discovered list + assert(java_lang_ref_Reference::discovered(obj) == NULL, "Already discovered"); + oop* const list = _discovered_list.addr(); + java_lang_ref_Reference::set_discovered(obj, *list); + *list = obj; +} + +oop ZReferenceProcessor::drop(oop obj, ReferenceType type) { + log_trace(gc, ref)("Dropped Reference: " PTR_FORMAT " (%s)", p2i(obj), reference_type_name(type)); + + // Keep referent alive + keep_referent_alive(obj, type); + + // Unlink and return next in list + const oop next = java_lang_ref_Reference::discovered(obj); + java_lang_ref_Reference::set_discovered(obj, NULL); + return next; +} + +oop* ZReferenceProcessor::keep(oop obj, ReferenceType type) { + log_trace(gc, ref)("Enqueued Reference: " PTR_FORMAT " (%s)", p2i(obj), reference_type_name(type)); + + // Update statistics + _enqueued_count.get()[type]++; + + // Clear referent + if (should_clear_referent(type)) { + java_lang_ref_Reference::set_referent(obj, NULL); + } + + // Make reference inactive by self-looping the next field. We could be racing with a + // call to Reference.enqueue() from the application, which is why we are using a CAS + // to make sure we change the next field only if it is NULL. A failing CAS means the + // reference has already been enqueued. However, we don't check the result of the CAS, + // since we still have no option other than keeping the reference on the pending list. + // It's ok to have the reference both on the pending list and enqueued at the same + // time (the pending list is linked through the discovered field, while the reference + // queue is linked through the next field). When the ReferenceHandler thread later + // calls Reference.enqueue() we detect that it has already been enqueued and drop it. + oop* const next_addr = (oop*)java_lang_ref_Reference::next_addr_raw(obj); + Atomic::cmpxchg(obj, next_addr, oop(NULL)); + + // Return next in list + return (oop*)java_lang_ref_Reference::discovered_addr_raw(obj); +} + +void ZReferenceProcessor::work() { + // Process discovered references + oop* const list = _discovered_list.addr(); + oop* p = list; + + while (*p != NULL) { + const oop obj = *p; + const ReferenceType type = reference_type(obj); + + if (should_drop_reference(obj, type)) { + *p = drop(obj, type); + } else { + p = keep(obj, type); + } + } + + // Prepend discovered references to internal pending list + if (*list != NULL) { + *p = Atomic::xchg(*list, _pending_list.addr()); + if (*p == NULL) { + // First to prepend to list, record tail + _pending_list_tail = p; + } + + // Clear discovered list + *list = NULL; + } +} + +bool ZReferenceProcessor::is_empty() const { + ZPerWorkerConstIterator iter(&_discovered_list); + for (const oop* list; iter.next(&list);) { + if (*list != NULL) { + return false; + } + } + + if (_pending_list.get() != NULL) { + return false; + } + + return true; +} + +void ZReferenceProcessor::reset_statistics() { + assert(is_empty(), "Should be empty"); + + // Reset encountered + ZPerWorkerIterator iter_encountered(&_encountered_count); + for (Counters* counters; iter_encountered.next(&counters);) { + for (int i = REF_SOFT; i <= REF_PHANTOM; i++) { + (*counters)[i] = 0; + } + } + + // Reset discovered + ZPerWorkerIterator iter_discovered(&_discovered_count); + for (Counters* counters; iter_discovered.next(&counters);) { + for (int i = REF_SOFT; i <= REF_PHANTOM; i++) { + (*counters)[i] = 0; + } + } + + // Reset enqueued + ZPerWorkerIterator iter_enqueued(&_enqueued_count); + for (Counters* counters; iter_enqueued.next(&counters);) { + for (int i = REF_SOFT; i <= REF_PHANTOM; i++) { + (*counters)[i] = 0; + } + } +} + +void ZReferenceProcessor::collect_statistics() { + Counters encountered = {}; + Counters discovered = {}; + Counters enqueued = {}; + + // Sum encountered + ZPerWorkerConstIterator iter_encountered(&_encountered_count); + for (const Counters* counters; iter_encountered.next(&counters);) { + for (int i = REF_SOFT; i <= REF_PHANTOM; i++) { + encountered[i] += (*counters)[i]; + } + } + + // Sum discovered + ZPerWorkerConstIterator iter_discovered(&_discovered_count); + for (const Counters* counters; iter_discovered.next(&counters);) { + for (int i = REF_SOFT; i <= REF_PHANTOM; i++) { + discovered[i] += (*counters)[i]; + } + } + + // Sum enqueued + ZPerWorkerConstIterator iter_enqueued(&_enqueued_count); + for (const Counters* counters; iter_enqueued.next(&counters);) { + for (int i = REF_SOFT; i <= REF_PHANTOM; i++) { + enqueued[i] += (*counters)[i]; + } + } + + // Update statistics + ZStatReferences::set_soft(encountered[REF_SOFT], discovered[REF_SOFT], enqueued[REF_SOFT]); + ZStatReferences::set_weak(encountered[REF_WEAK], discovered[REF_WEAK], enqueued[REF_WEAK]); + ZStatReferences::set_final(encountered[REF_FINAL], discovered[REF_FINAL], enqueued[REF_FINAL]); + ZStatReferences::set_phantom(encountered[REF_PHANTOM], discovered[REF_PHANTOM], enqueued[REF_PHANTOM]); + + // Trace statistics + const ReferenceProcessorStats stats(discovered[REF_SOFT], + discovered[REF_WEAK], + discovered[REF_FINAL], + discovered[REF_PHANTOM]); + ZTracer::tracer()->report_gc_reference_stats(stats); +} + +class ZReferenceProcessorTask : public ZTask { +private: + ZReferenceProcessor* const _reference_processor; + +public: + ZReferenceProcessorTask(ZReferenceProcessor* reference_processor) : + ZTask("ZReferenceProcessorTask"), + _reference_processor(reference_processor) {} + + virtual void work() { + _reference_processor->work(); + } +}; + +void ZReferenceProcessor::process_references() { + ZStatTimer timer(ZSubPhaseConcurrentReferencesProcess); + + // Process discovered lists + ZReferenceProcessorTask task(this); + _workers->run_concurrent(&task); + + // Update soft reference clock + update_soft_reference_clock(); + + // Collect, log and trace statistics + collect_statistics(); +} + +void ZReferenceProcessor::enqueue_references() { + ZStatTimer timer(ZSubPhaseConcurrentReferencesEnqueue); + + if (_pending_list.get() == NULL) { + // Nothing to enqueue + return; + } + + { + // Heap_lock protects external pending list + MonitorLockerEx ml(Heap_lock); + + // Prepend internal pending list to external pending list + *_pending_list_tail = Universe::swap_reference_pending_list(_pending_list.get()); + + // Notify ReferenceHandler thread + ml.notify_all(); + } + + // Reset internal pending list + _pending_list.set(NULL); + _pending_list_tail = _pending_list.addr(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zReferenceProcessor.hpp 2018-06-01 22:31:01.105765576 +0200 @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZREFERENCEPROCESSOR_HPP +#define SHARE_GC_Z_ZREFERENCEPROCESSOR_HPP + +#include "gc/shared/referenceDiscoverer.hpp" +#include "gc/z/zValue.hpp" + +class ReferencePolicy; +class ZWorkers; + +class ZReferenceProcessor : public ReferenceDiscoverer { + friend class ZReferenceProcessorTask; + +private: + static const size_t reference_type_count = REF_PHANTOM + 1; + typedef size_t Counters[reference_type_count]; + + ZWorkers* const _workers; + ReferencePolicy* _soft_reference_policy; + ZPerWorker _encountered_count; + ZPerWorker _discovered_count; + ZPerWorker _enqueued_count; + ZPerWorker _discovered_list; + ZContended _pending_list; + oop* _pending_list_tail; + + void update_soft_reference_clock() const; + + ReferenceType reference_type(oop obj) const; + const char* reference_type_name(ReferenceType type) const; + volatile oop* reference_referent_addr(oop obj) const; + oop reference_referent(oop obj) const; + bool is_reference_inactive(oop obj) const; + bool is_referent_alive_or_null(oop obj, ReferenceType type) const; + bool is_referent_softly_alive(oop obj, ReferenceType type) const; + bool should_drop_reference(oop obj, ReferenceType type) const; + bool should_mark_referent(ReferenceType type) const; + bool should_clear_referent(ReferenceType type) const; + void keep_referent_alive(oop obj, ReferenceType type) const; + + void discover(oop obj, ReferenceType type); + oop drop(oop obj, ReferenceType type); + oop* keep(oop obj, ReferenceType type); + + bool is_empty() const; + + void work(); + void collect_statistics(); + +public: + ZReferenceProcessor(ZWorkers* workers); + + void set_soft_reference_policy(bool clear); + void reset_statistics(); + + virtual bool discover_reference(oop reference, ReferenceType type); + void process_references(); + void enqueue_references(); +}; + +#endif // SHARE_GC_Z_ZREFERENCEPROCESSOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRelocate.cpp 2018-06-01 22:31:01.483781893 +0200 @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zHeap.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zPage.hpp" +#include "gc/z/zRelocate.hpp" +#include "gc/z/zRelocationSet.inline.hpp" +#include "gc/z/zRootsIterator.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zWorkers.hpp" + +ZRelocate::ZRelocate(ZWorkers* workers) : + _workers(workers) {} + +class ZRelocateRootsTask : public ZTask { +private: + ZRootsIterator _roots; + +public: + ZRelocateRootsTask() : + ZTask("ZRelocateRootsTask"), + _roots() {} + + virtual void work() { + // During relocation we need to visit the JVMTI + // export weak roots to rehash the JVMTI tag map + ZRelocateRootOopClosure cl; + _roots.oops_do(&cl, true /* visit_jvmti_weak_export */); + } +}; + +void ZRelocate::start() { + ZRelocateRootsTask task; + _workers->run_parallel(&task); +} + +class ZRelocateObjectClosure : public ObjectClosure { +private: + ZPage* const _page; + +public: + ZRelocateObjectClosure(ZPage* page) : + _page(page) {} + + virtual void do_object(oop o) { + _page->relocate_object(ZOop::to_address(o)); + } +}; + +bool ZRelocate::work(ZRelocationSetParallelIterator* iter) { + bool success = true; + + // Relocate pages in the relocation set + for (ZPage* page; iter->next(&page);) { + // Relocate objects in page + ZRelocateObjectClosure cl(page); + page->object_iterate(&cl); + + if (ZVerifyForwarding) { + page->verify_forwarding(); + } + + if (page->is_pinned()) { + // Relocation failed, page is now pinned + success = false; + } else { + // Relocation succeeded, release page + ZHeap::heap()->release_page(page, true /* reclaimed */); + } + } + + return success; +} + +class ZRelocateTask : public ZTask { +private: + ZRelocate* const _relocate; + ZRelocationSetParallelIterator _iter; + bool _failed; + +public: + ZRelocateTask(ZRelocate* relocate, ZRelocationSet* relocation_set) : + ZTask("ZRelocateTask"), + _relocate(relocate), + _iter(relocation_set), + _failed(false) {} + + virtual void work() { + if (!_relocate->work(&_iter)) { + _failed = true; + } + } + + bool failed() const { + return _failed; + } +}; + +bool ZRelocate::relocate(ZRelocationSet* relocation_set) { + ZRelocateTask task(this, relocation_set); + _workers->run_concurrent(&task); + return !task.failed(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRelocate.hpp 2018-06-01 22:31:01.866798425 +0200 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZRELOCATE_HPP +#define SHARE_GC_Z_ZRELOCATE_HPP + +#include "gc/z/zRelocationSet.hpp" +#include "gc/z/zWorkers.hpp" +#include "memory/allocation.hpp" + +class ZRelocate { + friend class ZRelocateTask; + +private: + ZWorkers* const _workers; + + bool work(ZRelocationSetParallelIterator* iter); + +public: + ZRelocate(ZWorkers* workers); + + void start(); + bool relocate(ZRelocationSet* relocation_set); +}; + +#endif // SHARE_GC_Z_ZRELOCATE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRelocationSet.cpp 2018-06-01 22:31:02.240814569 +0200 @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zRelocationSet.hpp" +#include "memory/allocation.inline.hpp" + +ZRelocationSet::ZRelocationSet() : + _pages(NULL), + _npages(0) {} + +void ZRelocationSet::populate(const ZPage* const* group0, size_t ngroup0, + const ZPage* const* group1, size_t ngroup1) { + _npages = ngroup0 + ngroup1; + _pages = REALLOC_C_HEAP_ARRAY(ZPage*, _pages, _npages, mtGC); + + if (_pages != NULL) { + if (group0 != NULL) { + memcpy(_pages, group0, ngroup0 * sizeof(ZPage*)); + } + if (group1 != NULL) { + memcpy(_pages + ngroup0, group1, ngroup1 * sizeof(ZPage*)); + } + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRelocationSet.hpp 2018-06-01 22:31:02.624831145 +0200 @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZRELOCATIONSET_HPP +#define SHARE_GC_Z_ZRELOCATIONSET_HPP + +#include "memory/allocation.hpp" + +class ZPage; + +class ZRelocationSet { + template friend class ZRelocationSetIteratorImpl; + +private: + ZPage** _pages; + size_t _npages; + +public: + ZRelocationSet(); + + void populate(const ZPage* const* group0, size_t ngroup0, + const ZPage* const* group1, size_t ngroup1); +}; + +template +class ZRelocationSetIteratorImpl : public StackObj { +private: + ZRelocationSet* const _relocation_set; + size_t _next; + +public: + ZRelocationSetIteratorImpl(ZRelocationSet* relocation_set); + + bool next(ZPage** page); +}; + +// Iterator types +#define ZRELOCATIONSET_SERIAL false +#define ZRELOCATIONSET_PARALLEL true + +class ZRelocationSetIterator : public ZRelocationSetIteratorImpl { +public: + ZRelocationSetIterator(ZRelocationSet* relocation_set) : + ZRelocationSetIteratorImpl(relocation_set) {} +}; + +class ZRelocationSetParallelIterator : public ZRelocationSetIteratorImpl { +public: + ZRelocationSetParallelIterator(ZRelocationSet* relocation_set) : + ZRelocationSetIteratorImpl(relocation_set) {} +}; + +#endif // SHARE_GC_Z_ZRELOCATIONSET_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRelocationSet.inline.hpp 2018-06-01 22:31:03.004847548 +0200 @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZRELOCATIONSET_INLINE_HPP +#define SHARE_GC_Z_ZRELOCATIONSET_INLINE_HPP + +#include "gc/z/zRelocationSet.hpp" +#include "runtime/atomic.hpp" + +template +inline ZRelocationSetIteratorImpl::ZRelocationSetIteratorImpl(ZRelocationSet* relocation_set) : + _relocation_set(relocation_set), + _next(0) {} + +template +inline bool ZRelocationSetIteratorImpl::next(ZPage** page) { + const size_t npages = _relocation_set->_npages; + + if (parallel) { + if (_next < npages) { + const size_t next = Atomic::add(1u, &_next) - 1u; + if (next < npages) { + *page = _relocation_set->_pages[next]; + return true; + } + } + } else { + if (_next < npages) { + *page = _relocation_set->_pages[_next++]; + return true; + } + } + + return false; +} + +#endif // SHARE_GC_Z_ZRELOCATIONSET_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRelocationSetSelector.cpp 2018-06-01 22:31:03.368863261 +0200 @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zArray.inline.hpp" +#include "gc/z/zPage.inline.hpp" +#include "gc/z/zRelocationSet.hpp" +#include "gc/z/zRelocationSetSelector.hpp" +#include "logging/log.hpp" +#include "runtime/globals.hpp" +#include "utilities/debug.hpp" + +ZRelocationSetSelectorGroup::ZRelocationSetSelectorGroup(const char* name, + size_t page_size, + size_t object_size_limit) : + _name(name), + _page_size(page_size), + _object_size_limit(object_size_limit), + _fragmentation_limit(page_size * (ZFragmentationLimit / 100)), + _registered_pages(), + _sorted_pages(NULL), + _nselected(0), + _relocating(0), + _fragmentation(0) {} + +ZRelocationSetSelectorGroup::~ZRelocationSetSelectorGroup() { + FREE_C_HEAP_ARRAY(const ZPage*, _sorted_pages); +} + +void ZRelocationSetSelectorGroup::register_live_page(const ZPage* page, size_t garbage) { + if (garbage > _fragmentation_limit) { + _registered_pages.add(page); + } else { + _fragmentation += garbage; + } +} + +void ZRelocationSetSelectorGroup::semi_sort() { + // Semi-sort registered pages by live bytes in ascending order + const size_t npartitions_shift = 11; + const size_t npartitions = (size_t)1 << npartitions_shift; + const size_t partition_size = _page_size >> npartitions_shift; + const size_t partition_size_shift = exact_log2(partition_size); + const size_t npages = _registered_pages.size(); + + size_t partition_slots[npartitions]; + size_t partition_finger[npartitions]; + + // Allocate destination array + _sorted_pages = REALLOC_C_HEAP_ARRAY(const ZPage*, _sorted_pages, npages, mtGC); + debug_only(memset(_sorted_pages, 0, npages * sizeof(ZPage*))); + + // Calculate partition slots + memset(partition_slots, 0, sizeof(partition_slots)); + ZArrayIterator iter1(&_registered_pages); + for (const ZPage* page; iter1.next(&page);) { + const size_t index = page->live_bytes() >> partition_size_shift; + partition_slots[index]++; + } + + // Calculate accumulated partition slots and fingers + size_t prev_partition_slots = 0; + for (size_t i = 0; i < npartitions; i++) { + partition_slots[i] += prev_partition_slots; + partition_finger[i] = prev_partition_slots; + prev_partition_slots = partition_slots[i]; + } + + // Sort pages into partitions + ZArrayIterator iter2(&_registered_pages); + for (const ZPage* page; iter2.next(&page);) { + const size_t index = page->live_bytes() >> partition_size_shift; + const size_t finger = partition_finger[index]++; + assert(_sorted_pages[finger] == NULL, "Invalid finger"); + _sorted_pages[finger] = page; + } +} + +void ZRelocationSetSelectorGroup::select() { + // Calculate the number of pages to relocate by successively including pages in + // a candidate relocation set and calculate the maximum space requirement for + // their live objects. + const size_t npages = _registered_pages.size(); + size_t selected_from = 0; + size_t selected_to = 0; + size_t from_size = 0; + + semi_sort(); + + for (size_t from = 1; from <= npages; from++) { + // Add page to the candidate relocation set + from_size += _sorted_pages[from - 1]->live_bytes(); + + // Calculate the maximum number of pages needed by the candidate relocation set. + // By subtracting the object size limit from the pages size we get the maximum + // number of pages that the relocation set is guaranteed to fit in, regardless + // of in which order the objects are relocated. + const size_t to = ceil((double)(from_size) / (double)(_page_size - _object_size_limit)); + + // Calculate the relative difference in reclaimable space compared to our + // currently selected final relocation set. If this number is larger than the + // acceptable fragmentation limit, then the current candidate relocation set + // becomes our new final relocation set. + const size_t diff_from = from - selected_from; + const size_t diff_to = to - selected_to; + const double diff_reclaimable = 100 - percent_of(diff_to, diff_from); + if (diff_reclaimable > ZFragmentationLimit) { + selected_from = from; + selected_to = to; + } + + log_trace(gc, reloc)("Candidate Relocation Set (%s Pages): " + SIZE_FORMAT "->" SIZE_FORMAT ", %.1f%% relative defragmentation, %s", + _name, from, to, diff_reclaimable, (selected_from == from) ? "Selected" : "Rejected"); + } + + // Finalize selection + _nselected = selected_from; + + // Update statistics + _relocating = from_size; + for (size_t i = _nselected; i < npages; i++) { + const ZPage* const page = _sorted_pages[i]; + _fragmentation += page->size() - page->live_bytes(); + } + + log_debug(gc, reloc)("Relocation Set (%s Pages): " SIZE_FORMAT "->" SIZE_FORMAT ", " SIZE_FORMAT " skipped", + _name, selected_from, selected_to, npages - _nselected); +} + +const ZPage* const* ZRelocationSetSelectorGroup::selected() const { + return _sorted_pages; +} + +size_t ZRelocationSetSelectorGroup::nselected() const { + return _nselected; +} + +size_t ZRelocationSetSelectorGroup::relocating() const { + return _relocating; +} + +size_t ZRelocationSetSelectorGroup::fragmentation() const { + return _fragmentation; +} + +ZRelocationSetSelector::ZRelocationSetSelector() : + _small("Small", ZPageSizeSmall, ZObjectSizeLimitSmall), + _medium("Medium", ZPageSizeMedium, ZObjectSizeLimitMedium), + _live(0), + _garbage(0), + _fragmentation(0) {} + +void ZRelocationSetSelector::register_live_page(const ZPage* page) { + const uint8_t type = page->type(); + const size_t live = page->live_bytes(); + const size_t garbage = page->size() - live; + + if (type == ZPageTypeSmall) { + _small.register_live_page(page, garbage); + } else if (type == ZPageTypeMedium) { + _medium.register_live_page(page, garbage); + } else { + _fragmentation += garbage; + } + + _live += live; + _garbage += garbage; +} + +void ZRelocationSetSelector::register_garbage_page(const ZPage* page) { + _garbage += page->size(); +} + +void ZRelocationSetSelector::select(ZRelocationSet* relocation_set) { + // Select pages to relocate. The resulting relocation set will be + // sorted such that medium pages comes first, followed by small + // pages. Pages within each page group will be semi-sorted by live + // bytes in ascending order. Relocating pages in this order allows + // us to start reclaiming memory more quickly. + + // Select pages from each group + _medium.select(); + _small.select(); + + // Populate relocation set + relocation_set->populate(_medium.selected(), _medium.nselected(), + _small.selected(), _small.nselected()); +} + +size_t ZRelocationSetSelector::live() const { + return _live; +} + +size_t ZRelocationSetSelector::garbage() const { + return _garbage; +} + +size_t ZRelocationSetSelector::relocating() const { + return _small.relocating() + _medium.relocating(); +} + +size_t ZRelocationSetSelector::fragmentation() const { + return _fragmentation + _small.fragmentation() + _medium.fragmentation(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRelocationSetSelector.hpp 2018-06-01 22:31:03.749879707 +0200 @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZRELOCATIONSETSELECTOR_HPP +#define SHARE_GC_Z_ZRELOCATIONSETSELECTOR_HPP + +#include "gc/z/zArray.hpp" +#include "memory/allocation.hpp" + +class ZPage; +class ZRelocationSet; + +class ZRelocationSetSelectorGroup { +private: + const char* const _name; + const size_t _page_size; + const size_t _object_size_limit; + const size_t _fragmentation_limit; + + ZArray _registered_pages; + const ZPage** _sorted_pages; + size_t _nselected; + size_t _relocating; + size_t _fragmentation; + + void semi_sort(); + +public: + ZRelocationSetSelectorGroup(const char* name, + size_t page_size, + size_t object_size_limit); + ~ZRelocationSetSelectorGroup(); + + void register_live_page(const ZPage* page, size_t garbage); + void select(); + + const ZPage* const* selected() const; + size_t nselected() const; + size_t relocating() const; + size_t fragmentation() const; +}; + +class ZRelocationSetSelector : public StackObj { +private: + ZRelocationSetSelectorGroup _small; + ZRelocationSetSelectorGroup _medium; + size_t _live; + size_t _garbage; + size_t _fragmentation; + +public: + ZRelocationSetSelector(); + + void register_live_page(const ZPage* page); + void register_garbage_page(const ZPage* page); + void select(ZRelocationSet* relocation_set); + + size_t live() const; + size_t garbage() const; + size_t relocating() const; + size_t fragmentation() const; +}; + +#endif // SHARE_GC_Z_ZRELOCATIONSETSELECTOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zResurrection.cpp 2018-06-01 22:31:04.126895980 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zResurrection.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/safepoint.hpp" +#include "utilities/debug.hpp" + +volatile bool ZResurrection::_blocked = false; + +void ZResurrection::block() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + _blocked = true; +} + +void ZResurrection::unblock() { + // We use a storestore barrier to make sure all healed + // oops are visible before we unblock resurrection. + OrderAccess::storestore(); + _blocked = false; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zResurrection.hpp 2018-06-01 22:31:04.512912643 +0200 @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZRESURRECTION_HPP +#define SHARE_GC_Z_ZRESURRECTION_HPP + +#include "memory/allocation.hpp" + +class ZResurrection : public AllStatic { +private: + static volatile bool _blocked; + +public: + static bool is_blocked(); + static void block(); + static void unblock(); +}; + +#endif // SHARE_GC_Z_ZRESURRECTION_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zResurrection.inline.hpp 2018-06-01 22:31:04.904929564 +0200 @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZRESURRECTION_INLINE_HPP +#define SHARE_GC_Z_ZRESURRECTION_INLINE_HPP + +#include "gc/z/zResurrection.hpp" +#include "runtime/orderAccess.inline.hpp" + +inline bool ZResurrection::is_blocked() { + // We use a loadload barrier to make sure we are not + // seeing oops from a time when resurrection was blocked. + const bool blocked = _blocked; + OrderAccess::loadload(); + return blocked; +} + +#endif // SHARE_GC_Z_ZRESURRECTION_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRootsIterator.cpp 2018-06-01 22:31:05.288946139 +0200 @@ -0,0 +1,421 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "classfile/stringTable.hpp" +#include "classfile/symbolTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "compiler/oopMap.hpp" +#include "gc/shared/oopStorageParState.inline.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zRootsIterator.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zThreadLocalData.hpp" +#include "memory/resourceArea.hpp" +#include "memory/universe.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/atomic.hpp" +#include "runtime/jniHandles.hpp" +#include "runtime/thread.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/synchronizer.hpp" +#include "services/management.hpp" +#include "utilities/debug.hpp" +#if INCLUDE_JFR +#include "jfr/jfr.hpp" +#endif + +static const ZStatSubPhase ZSubPhasePauseRootsSetup("Pause Roots Setup"); +static const ZStatSubPhase ZSubPhasePauseRoots("Pause Roots"); +static const ZStatSubPhase ZSubPhasePauseRootsTeardown("Pause Roots Teardown"); +static const ZStatSubPhase ZSubPhasePauseRootsUniverse("Pause Roots Universe"); +static const ZStatSubPhase ZSubPhasePauseRootsVMWeakHandles("Pause Roots VMWeakHandles"); +static const ZStatSubPhase ZSubPhasePauseRootsJNIHandles("Pause Roots JNIHandles"); +static const ZStatSubPhase ZSubPhasePauseRootsJNIWeakHandles("Pause Roots JNIWeakHandles"); +static const ZStatSubPhase ZSubPhasePauseRootsObjectSynchronizer("Pause Roots ObjectSynchronizer"); +static const ZStatSubPhase ZSubPhasePauseRootsManagement("Pause Roots Management"); +static const ZStatSubPhase ZSubPhasePauseRootsJVMTIExport("Pause Roots JVMTIExport"); +static const ZStatSubPhase ZSubPhasePauseRootsJVMTIWeakExport("Pause Roots JVMTIWeakExport"); +static const ZStatSubPhase ZSubPhasePauseRootsJFRWeak("Pause Roots JRFWeak"); +static const ZStatSubPhase ZSubPhasePauseRootsSystemDictionary("Pause Roots SystemDictionary"); +static const ZStatSubPhase ZSubPhasePauseRootsClassLoaderDataGraph("Pause Roots ClassLoaderDataGraph"); +static const ZStatSubPhase ZSubPhasePauseRootsThreads("Pause Roots Threads"); +static const ZStatSubPhase ZSubPhasePauseRootsCodeCache("Pause Roots CodeCache"); +static const ZStatSubPhase ZSubPhasePauseRootsStringTable("Pause Roots StringTable"); + +static const ZStatSubPhase ZSubPhasePauseWeakRootsSetup("Pause Weak Roots Setup"); +static const ZStatSubPhase ZSubPhasePauseWeakRoots("Pause Weak Roots"); +static const ZStatSubPhase ZSubPhasePauseWeakRootsTeardown("Pause Weak Roots Teardown"); +static const ZStatSubPhase ZSubPhasePauseWeakRootsVMWeakHandles("Pause Weak Roots VMWeakHandles"); +static const ZStatSubPhase ZSubPhasePauseWeakRootsJNIWeakHandles("Pause Weak Roots JNIWeakHandles"); +static const ZStatSubPhase ZSubPhasePauseWeakRootsJVMTIWeakExport("Pause Weak Roots JVMTIWeakExport"); +static const ZStatSubPhase ZSubPhasePauseWeakRootsJFRWeak("Pause Weak Roots JFRWeak"); +static const ZStatSubPhase ZSubPhasePauseWeakRootsSymbolTable("Pause Weak Roots SymbolTable"); +static const ZStatSubPhase ZSubPhasePauseWeakRootsStringTable("Pause Weak Roots StringTable"); + +static const ZStatSubPhase ZSubPhaseConcurrentWeakRoots("Concurrent Weak Roots"); +static const ZStatSubPhase ZSubPhaseConcurrentWeakRootsVMWeakHandles("Concurrent Weak Roots VMWeakHandles"); +static const ZStatSubPhase ZSubPhaseConcurrentWeakRootsJNIWeakHandles("Concurrent Weak Roots JNIWeakHandles"); + +template +ZSerialOopsDo::ZSerialOopsDo(T* iter) : + _iter(iter), + _claimed(false) {} + +template +void ZSerialOopsDo::oops_do(OopClosure* cl) { + if (!_claimed && Atomic::cmpxchg(true, &_claimed, false) == false) { + (_iter->*F)(cl); + } +} + +template +ZParallelOopsDo::ZParallelOopsDo(T* iter) : + _iter(iter), + _completed(false) {} + +template +void ZParallelOopsDo::oops_do(OopClosure* cl) { + if (!_completed) { + (_iter->*F)(cl); + if (!_completed) { + _completed = true; + } + } +} + +template +ZSerialWeakOopsDo::ZSerialWeakOopsDo(T* iter) : + _iter(iter), + _claimed(false) {} + +template +void ZSerialWeakOopsDo::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* cl) { + if (!_claimed && Atomic::cmpxchg(true, &_claimed, false) == false) { + (_iter->*F)(is_alive, cl); + } +} + +template +ZParallelWeakOopsDo::ZParallelWeakOopsDo(T* iter) : + _iter(iter), + _completed(false) {} + +template +void ZParallelWeakOopsDo::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* cl) { + if (!_completed) { + (_iter->*F)(is_alive, cl); + if (!_completed) { + _completed = true; + } + } +} + +ZRootsIterator::ZRootsIterator() : + _vm_weak_handles_iter(SystemDictionary::vm_weak_oop_storage()), + _jni_handles_iter(JNIHandles::global_handles()), + _jni_weak_handles_iter(JNIHandles::weak_global_handles()), + _universe(this), + _object_synchronizer(this), + _management(this), + _jvmti_export(this), + _jvmti_weak_export(this), + _jfr_weak(this), + _system_dictionary(this), + _vm_weak_handles(this), + _jni_handles(this), + _jni_weak_handles(this), + _class_loader_data_graph(this), + _threads(this), + _code_cache(this), + _string_table(this) { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + ZStatTimer timer(ZSubPhasePauseRootsSetup); + Threads::change_thread_claim_parity(); + StringTable::clear_parallel_claimed_index(); + ClassLoaderDataGraph::clear_claimed_marks(); + COMPILER2_PRESENT(DerivedPointerTable::clear()); + CodeCache::gc_prologue(); + ZNMethodTable::gc_prologue(); +} + +ZRootsIterator::~ZRootsIterator() { + ZStatTimer timer(ZSubPhasePauseRootsTeardown); + ResourceMark rm; + ZNMethodTable::gc_epilogue(); + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + Threads::assert_all_threads_claimed(); +} + +void ZRootsIterator::do_universe(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsUniverse); + Universe::oops_do(cl); +} + +void ZRootsIterator::do_vm_weak_handles(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsVMWeakHandles); + _vm_weak_handles_iter.oops_do(cl); +} + +void ZRootsIterator::do_jni_handles(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsJNIHandles); + _jni_handles_iter.oops_do(cl); +} + +void ZRootsIterator::do_jni_weak_handles(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsJNIWeakHandles); + _jni_weak_handles_iter.oops_do(cl); +} + +void ZRootsIterator::do_object_synchronizer(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsObjectSynchronizer); + ObjectSynchronizer::oops_do(cl); +} + +void ZRootsIterator::do_management(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsManagement); + Management::oops_do(cl); +} + +void ZRootsIterator::do_jvmti_export(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsJVMTIExport); + JvmtiExport::oops_do(cl); +} + +void ZRootsIterator::do_jvmti_weak_export(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsJVMTIWeakExport); + AlwaysTrueClosure always_alive; + JvmtiExport::weak_oops_do(&always_alive, cl); +} + +void ZRootsIterator::do_jfr_weak(OopClosure* cl) { +#if INCLUDE_JFR + ZStatTimer timer(ZSubPhasePauseRootsJFRWeak); + AlwaysTrueClosure always_alive; + Jfr::weak_oops_do(&always_alive, cl); +#endif +} + +void ZRootsIterator::do_system_dictionary(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsSystemDictionary); + SystemDictionary::oops_do(cl); +} + +void ZRootsIterator::do_class_loader_data_graph(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsClassLoaderDataGraph); + CLDToOopClosure cld_cl(cl); + ClassLoaderDataGraph::cld_do(&cld_cl); +} + +class ZRootsIteratorThreadClosure : public ThreadClosure { +private: + OopClosure* const _cl; + +public: + ZRootsIteratorThreadClosure(OopClosure* cl) : + _cl(cl) {} + + virtual void do_thread(Thread* thread) { + if (thread->is_Java_thread()) { + // Update thread local adddress bad mask + ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask); + } + + // Process thread oops + thread->oops_do(_cl, NULL); + } +}; + +void ZRootsIterator::do_threads(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsThreads); + ResourceMark rm; + ZRootsIteratorThreadClosure thread_cl(cl); + Threads::possibly_parallel_threads_do(true, &thread_cl); +} + +void ZRootsIterator::do_code_cache(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsCodeCache); + ZNMethodTable::oops_do(cl); +} + +void ZRootsIterator::do_string_table(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsStringTable); + StringTable::possibly_parallel_oops_do(cl); +} + +void ZRootsIterator::oops_do(OopClosure* cl, bool visit_jvmti_weak_export) { + ZStatTimer timer(ZSubPhasePauseRoots); + _universe.oops_do(cl); + _object_synchronizer.oops_do(cl); + _management.oops_do(cl); + _jvmti_export.oops_do(cl); + _system_dictionary.oops_do(cl); + _jni_handles.oops_do(cl); + _class_loader_data_graph.oops_do(cl); + _threads.oops_do(cl); + _code_cache.oops_do(cl); + if (!ZWeakRoots) { + _jvmti_weak_export.oops_do(cl); + _jfr_weak.oops_do(cl); + _vm_weak_handles.oops_do(cl); + _jni_weak_handles.oops_do(cl); + _string_table.oops_do(cl); + } else { + if (visit_jvmti_weak_export) { + _jvmti_weak_export.oops_do(cl); + } + } +} + +ZWeakRootsIterator::ZWeakRootsIterator() : + _vm_weak_handles_iter(SystemDictionary::vm_weak_oop_storage()), + _jni_weak_handles_iter(JNIHandles::weak_global_handles()), + _jvmti_weak_export(this), + _jfr_weak(this), + _vm_weak_handles(this), + _jni_weak_handles(this), + _symbol_table(this), + _string_table(this) { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + ZStatTimer timer(ZSubPhasePauseWeakRootsSetup); + SymbolTable::clear_parallel_claimed_index(); + StringTable::clear_parallel_claimed_index(); +} + +ZWeakRootsIterator::~ZWeakRootsIterator() { + ZStatTimer timer(ZSubPhasePauseWeakRootsTeardown); +} + +void ZWeakRootsIterator::do_vm_weak_handles(BoolObjectClosure* is_alive, OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseWeakRootsVMWeakHandles); + _vm_weak_handles_iter.weak_oops_do(is_alive, cl); +} + +void ZWeakRootsIterator::do_jni_weak_handles(BoolObjectClosure* is_alive, OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseWeakRootsJNIWeakHandles); + _jni_weak_handles_iter.weak_oops_do(is_alive, cl); +} + +void ZWeakRootsIterator::do_jvmti_weak_export(BoolObjectClosure* is_alive, OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseWeakRootsJVMTIWeakExport); + JvmtiExport::weak_oops_do(is_alive, cl); +} + +void ZWeakRootsIterator::do_jfr_weak(BoolObjectClosure* is_alive, OopClosure* cl) { +#if INCLUDE_JFR + ZStatTimer timer(ZSubPhasePauseWeakRootsJFRWeak); + Jfr::weak_oops_do(is_alive, cl); +#endif +} + +void ZWeakRootsIterator::do_symbol_table(BoolObjectClosure* is_alive, OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseWeakRootsSymbolTable); + int dummy; + SymbolTable::possibly_parallel_unlink(&dummy, &dummy); +} + +void ZWeakRootsIterator::do_string_table(BoolObjectClosure* is_alive, OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseWeakRootsStringTable); + int dummy; + StringTable::possibly_parallel_unlink_or_oops_do(is_alive, cl, &dummy, &dummy); +} + +void ZWeakRootsIterator::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseWeakRoots); + if (ZSymbolTableUnloading) { + _symbol_table.weak_oops_do(is_alive, cl); + } + if (ZWeakRoots) { + _jvmti_weak_export.weak_oops_do(is_alive, cl); + _jfr_weak.weak_oops_do(is_alive, cl); + if (!ZConcurrentVMWeakHandles) { + _vm_weak_handles.weak_oops_do(is_alive, cl); + } + if (!ZConcurrentJNIWeakGlobalHandles) { + _jni_weak_handles.weak_oops_do(is_alive, cl); + } + _string_table.weak_oops_do(is_alive, cl); + } +} + +void ZWeakRootsIterator::oops_do(OopClosure* cl) { + AlwaysTrueClosure always_alive; + weak_oops_do(&always_alive, cl); +} + +ZConcurrentWeakRootsIterator::ZConcurrentWeakRootsIterator() : + _vm_weak_handles_iter(SystemDictionary::vm_weak_oop_storage()), + _jni_weak_handles_iter(JNIHandles::weak_global_handles()), + _vm_weak_handles(this), + _jni_weak_handles(this) {} + +void ZConcurrentWeakRootsIterator::do_vm_weak_handles(OopClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentWeakRootsVMWeakHandles); + _vm_weak_handles_iter.oops_do(cl); +} + +void ZConcurrentWeakRootsIterator::do_jni_weak_handles(OopClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentWeakRootsJNIWeakHandles); + _jni_weak_handles_iter.oops_do(cl); +} + +void ZConcurrentWeakRootsIterator::oops_do(OopClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentWeakRoots); + if (ZWeakRoots) { + if (ZConcurrentVMWeakHandles) { + _vm_weak_handles.oops_do(cl); + } + if (ZConcurrentJNIWeakGlobalHandles) { + _jni_weak_handles.oops_do(cl); + } + } +} + +ZThreadRootsIterator::ZThreadRootsIterator() : + _threads(this) { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); + ZStatTimer timer(ZSubPhasePauseRootsSetup); + Threads::change_thread_claim_parity(); +} + +ZThreadRootsIterator::~ZThreadRootsIterator() { + ZStatTimer timer(ZSubPhasePauseRootsTeardown); + Threads::assert_all_threads_claimed(); +} + +void ZThreadRootsIterator::do_threads(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRootsThreads); + ResourceMark rm; + Threads::possibly_parallel_oops_do(true, cl, NULL); +} + +void ZThreadRootsIterator::oops_do(OopClosure* cl) { + ZStatTimer timer(ZSubPhasePauseRoots); + _threads.oops_do(cl); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRootsIterator.hpp 2018-06-01 22:31:05.663962327 +0200 @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZROOTSITERATOR_HPP +#define SHARE_GC_Z_ZROOTSITERATOR_HPP + +#include "gc/shared/oopStorageParState.hpp" +#include "memory/allocation.hpp" +#include "memory/iterator.hpp" +#include "utilities/globalDefinitions.hpp" + +typedef OopStorage::ParState ZOopStorageIterator; +typedef OopStorage::ParState ZConcurrentOopStorageIterator; + +template +class ZSerialOopsDo { +private: + T* const _iter; + volatile bool _claimed; + +public: + ZSerialOopsDo(T* iter); + void oops_do(OopClosure* cl); +}; + +template +class ZParallelOopsDo { +private: + T* const _iter; + volatile bool _completed; + +public: + ZParallelOopsDo(T* iter); + void oops_do(OopClosure* cl); +}; + +template +class ZSerialWeakOopsDo { +private: + T* const _iter; + volatile bool _claimed; + +public: + ZSerialWeakOopsDo(T* iter); + void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* cl); +}; + +template +class ZParallelWeakOopsDo { +private: + T* const _iter; + volatile bool _completed; + +public: + ZParallelWeakOopsDo(T* iter); + void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* cl); +}; + +class ZRootsIterator { +private: + ZOopStorageIterator _vm_weak_handles_iter; + ZOopStorageIterator _jni_handles_iter; + ZOopStorageIterator _jni_weak_handles_iter; + + void do_universe(OopClosure* cl); + void do_vm_weak_handles(OopClosure* cl); + void do_jni_handles(OopClosure* cl); + void do_jni_weak_handles(OopClosure* cl); + void do_object_synchronizer(OopClosure* cl); + void do_management(OopClosure* cl); + void do_jvmti_export(OopClosure* cl); + void do_jvmti_weak_export(OopClosure* cl); + void do_jfr_weak(OopClosure* cl); + void do_system_dictionary(OopClosure* cl); + void do_class_loader_data_graph(OopClosure* cl); + void do_threads(OopClosure* cl); + void do_code_cache(OopClosure* cl); + void do_string_table(OopClosure* cl); + + ZSerialOopsDo _universe; + ZSerialOopsDo _object_synchronizer; + ZSerialOopsDo _management; + ZSerialOopsDo _jvmti_export; + ZSerialOopsDo _jvmti_weak_export; + ZSerialOopsDo _jfr_weak; + ZSerialOopsDo _system_dictionary; + ZParallelOopsDo _vm_weak_handles; + ZParallelOopsDo _jni_handles; + ZParallelOopsDo _jni_weak_handles; + ZParallelOopsDo _class_loader_data_graph; + ZParallelOopsDo _threads; + ZParallelOopsDo _code_cache; + ZParallelOopsDo _string_table; + +public: + ZRootsIterator(); + ~ZRootsIterator(); + + void oops_do(OopClosure* cl, bool visit_jvmti_weak_export = false); +}; + +class ZWeakRootsIterator { +private: + ZOopStorageIterator _vm_weak_handles_iter; + ZOopStorageIterator _jni_weak_handles_iter; + + void do_vm_weak_handles(BoolObjectClosure* is_alive, OopClosure* cl); + void do_jni_weak_handles(BoolObjectClosure* is_alive, OopClosure* cl); + void do_jvmti_weak_export(BoolObjectClosure* is_alive, OopClosure* cl); + void do_jfr_weak(BoolObjectClosure* is_alive, OopClosure* cl); + void do_symbol_table(BoolObjectClosure* is_alive, OopClosure* cl); + void do_string_table(BoolObjectClosure* is_alive, OopClosure* cl); + + ZSerialWeakOopsDo _jvmti_weak_export; + ZSerialWeakOopsDo _jfr_weak; + ZParallelWeakOopsDo _vm_weak_handles; + ZParallelWeakOopsDo _jni_weak_handles; + ZParallelWeakOopsDo _symbol_table; + ZParallelWeakOopsDo _string_table; + +public: + ZWeakRootsIterator(); + ~ZWeakRootsIterator(); + + void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* cl); + void oops_do(OopClosure* cl); +}; + +class ZConcurrentWeakRootsIterator { +private: + ZConcurrentOopStorageIterator _vm_weak_handles_iter; + ZConcurrentOopStorageIterator _jni_weak_handles_iter; + + void do_vm_weak_handles(OopClosure* cl); + void do_jni_weak_handles(OopClosure* cl); + + ZParallelOopsDo _vm_weak_handles; + ZParallelOopsDo _jni_weak_handles; + +public: + ZConcurrentWeakRootsIterator(); + + void oops_do(OopClosure* cl); +}; + +class ZThreadRootsIterator { +private: + void do_threads(OopClosure* cl); + + ZParallelOopsDo _threads; + +public: + ZThreadRootsIterator(); + ~ZThreadRootsIterator(); + + void oops_do(OopClosure* cl); +}; + +#endif // SHARE_GC_Z_ZROOTSITERATOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRuntimeWorkers.cpp 2018-06-01 22:31:06.043978730 +0200 @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zRuntimeWorkers.hpp" + +ZRuntimeWorkers::ZRuntimeWorkers() : + _workers("RuntimeWorker", + nworkers(), + false /* are_GC_task_threads */, + false /* are_ConcurrentGC_threads */) { + + log_info(gc, init)("Runtime Workers: %u parallel", nworkers()); + + // Initialize worker threads + _workers.initialize_workers(); + _workers.update_active_workers(nworkers()); +} + +uint ZRuntimeWorkers::nworkers() const { + return ParallelGCThreads; +} + +WorkGang* ZRuntimeWorkers::workers() { + return &_workers; +} + +void ZRuntimeWorkers::threads_do(ThreadClosure* tc) const { + _workers.threads_do(tc); +} + +void ZRuntimeWorkers::print_threads_on(outputStream* st) const { + _workers.print_worker_threads_on(st); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zRuntimeWorkers.hpp 2018-06-01 22:31:06.435995651 +0200 @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZRUNTIMEWORKERS_HPP +#define SHARE_GC_Z_ZRUNTIMEWORKERS_HPP + +#include "gc/shared/workgroup.hpp" + +class ZRuntimeWorkers { +private: + WorkGang _workers; + + uint nworkers() const; + +public: + ZRuntimeWorkers(); + + WorkGang* workers(); + + void threads_do(ThreadClosure* tc) const; + void print_threads_on(outputStream* st) const; +}; + +#endif // SHARE_GC_Z_ZRUNTIMEWORKERS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zServiceability.cpp 2018-06-01 22:31:06.826012485 +0200 @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/generationCounters.hpp" +#include "gc/shared/hSpaceCounters.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zServiceability.hpp" +#include "memory/metaspaceCounters.hpp" +#include "runtime/perfData.hpp" + +class ZOldGenerationCounters : public GenerationCounters { +public: + ZOldGenerationCounters(const char* name, size_t min_capacity, size_t max_capacity) : + // The "1, 1" parameters are for the n-th generation (=1) with 1 space. + GenerationCounters(name, + 1 /* ordinal */, + 1 /* spaces */, + min_capacity /* min_capacity */, + max_capacity /* max_capacity */, + min_capacity /* curr_capacity */) {} + + virtual void update_all() { + size_t committed = ZHeap::heap()->capacity(); + _current_size->set_value(committed); + } +}; + +// Class to expose perf counters used by jstat. +class ZServiceabilityCounters : public CHeapObj { +private: + ZOldGenerationCounters _old_collection_counters; + HSpaceCounters _old_space_counters; + +public: + ZServiceabilityCounters(size_t min_capacity, size_t max_capacity); + + void update_sizes(); +}; + +ZServiceabilityCounters::ZServiceabilityCounters(size_t min_capacity, size_t max_capacity) : + // generation.1 + _old_collection_counters("old", + min_capacity, + max_capacity), + // generation.1.space.0 + _old_space_counters(_old_collection_counters.name_space(), + "space", + 0 /* ordinal */, + max_capacity /* max_capacity */, + min_capacity /* init_capacity */) {} + +void ZServiceabilityCounters::update_sizes() { + if (UsePerfData) { + size_t capacity = ZHeap::heap()->capacity(); + size_t used = MIN2(ZHeap::heap()->used(), capacity); + + _old_space_counters.update_capacity(capacity); + _old_space_counters.update_used(used); + + _old_collection_counters.update_all(); + + MetaspaceCounters::update_performance_counters(); + CompressedClassSpaceCounters::update_performance_counters(); + } +} + +ZServiceabilityMemoryPool::ZServiceabilityMemoryPool(size_t min_capacity, size_t max_capacity) : + CollectedMemoryPool("ZHeap", + min_capacity, + max_capacity, + true /* support_usage_threshold */) {} + +size_t ZServiceabilityMemoryPool::used_in_bytes() { + return ZHeap::heap()->used(); +} + +MemoryUsage ZServiceabilityMemoryPool::get_memory_usage() { + const size_t committed = ZHeap::heap()->capacity(); + const size_t used = MIN2(ZHeap::heap()->used(), committed); + + return MemoryUsage(initial_size(), used, committed, max_size()); +} + +ZServiceabilityMemoryManager::ZServiceabilityMemoryManager(ZServiceabilityMemoryPool* pool) + : GCMemoryManager("ZGC", "end of major GC") { + add_pool(pool); +} + +ZServiceability::ZServiceability(size_t min_capacity, size_t max_capacity) : + _min_capacity(min_capacity), + _max_capacity(max_capacity), + _memory_pool(_min_capacity, _max_capacity), + _memory_manager(&_memory_pool), + _counters(NULL) {} + +void ZServiceability::initialize() { + _counters = new ZServiceabilityCounters(_min_capacity, _max_capacity); +} + +MemoryPool* ZServiceability::memory_pool() { + return &_memory_pool; +} + +GCMemoryManager* ZServiceability::memory_manager() { + return &_memory_manager; +} + +ZServiceabilityCounters* ZServiceability::counters() { + return _counters; +} + +ZServiceabilityMemoryUsageTracker::~ZServiceabilityMemoryUsageTracker() { + MemoryService::track_memory_usage(); +} + +ZServiceabilityManagerStatsTracer::ZServiceabilityManagerStatsTracer(bool is_gc_begin, bool is_gc_end) : + _stats(ZHeap::heap()->serviceability_memory_manager(), + ZCollectedHeap::heap()->gc_cause() /* cause */, + is_gc_begin /* recordGCBeginTime */, + is_gc_begin /* recordPreGCUsage */, + true /* recordPeakUsage */, + is_gc_end /* recordPostGCusage */, + true /* recordAccumulatedGCTime */, + is_gc_end /* recordGCEndTime */, + is_gc_end /* countCollection */) {} + +ZServiceabilityCountersTracer::ZServiceabilityCountersTracer() { + // Nothing to trace with TraceCollectorStats, since ZGC has + // neither a young collector or a full collector. +} + +ZServiceabilityCountersTracer::~ZServiceabilityCountersTracer() { + ZHeap::heap()->serviceability_counters()->update_sizes(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zServiceability.hpp 2018-06-01 22:31:07.201028673 +0200 @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZSERVICEABILITY_HPP +#define SHARE_GC_Z_ZSERVICEABILITY_HPP + +#include "memory/allocation.hpp" +#include "services/memoryManager.hpp" +#include "services/memoryPool.hpp" +#include "services/memoryService.hpp" + +class ZServiceabilityCounters; + +class ZServiceabilityMemoryPool : public CollectedMemoryPool { +public: + ZServiceabilityMemoryPool(size_t min_capacity, size_t max_capacity); + + virtual size_t used_in_bytes(); + virtual MemoryUsage get_memory_usage(); +}; + +class ZServiceabilityMemoryManager : public GCMemoryManager { +public: + ZServiceabilityMemoryManager(ZServiceabilityMemoryPool* pool); +}; + +class ZServiceability { +private: + const size_t _min_capacity; + const size_t _max_capacity; + ZServiceabilityMemoryPool _memory_pool; + ZServiceabilityMemoryManager _memory_manager; + ZServiceabilityCounters* _counters; + +public: + ZServiceability(size_t min_capacity, size_t max_capacity); + + void initialize(); + + MemoryPool* memory_pool(); + GCMemoryManager* memory_manager(); + ZServiceabilityCounters* counters(); +}; + +class ZServiceabilityMemoryUsageTracker { +public: + ~ZServiceabilityMemoryUsageTracker(); +}; + +class ZServiceabilityManagerStatsTracer { +private: + TraceMemoryManagerStats _stats; + +public: + ZServiceabilityManagerStatsTracer(bool is_gc_begin, bool is_gc_end); +}; + +class ZServiceabilityCountersTracer { +public: + ZServiceabilityCountersTracer(); + ~ZServiceabilityCountersTracer(); +}; + +template +class ZServiceabilityTracer : public StackObj { +private: + ZServiceabilityMemoryUsageTracker _memory_usage_tracker; + ZServiceabilityManagerStatsTracer _manager_stats_tracer; + ZServiceabilityCountersTracer _counters_tracer; + +public: + ZServiceabilityTracer() : + _memory_usage_tracker(), + _manager_stats_tracer(IsGCStart, IsGCEnd), + _counters_tracer() {} +}; + +typedef ZServiceabilityTracer ZServiceabilityMarkStartTracer; +typedef ZServiceabilityTracer ZServiceabilityMarkEndTracer; +typedef ZServiceabilityTracer ZServiceabilityRelocateStartTracer; + +#endif // SHARE_GC_Z_ZSERVICEABILITY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zStat.cpp 2018-06-01 22:31:07.572044687 +0200 @@ -0,0 +1,1366 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zCollectedHeap.hpp" +#include "gc/z/zCPU.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zLargePages.inline.hpp" +#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zNUMA.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTracer.inline.hpp" +#include "gc/z/zUtils.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/atomic.hpp" +#include "runtime/os.hpp" +#include "runtime/timer.hpp" +#include "utilities/align.hpp" +#include "utilities/compilerWarnings.hpp" +#include "utilities/debug.hpp" +#include "utilities/ticks.hpp" + +// +// Stat sampler/counter data +// +struct ZStatSamplerData { + uint64_t _nsamples; + uint64_t _sum; + uint64_t _max; + + ZStatSamplerData() : + _nsamples(0), + _sum(0), + _max(0) {} + + void add(const ZStatSamplerData& new_sample) { + _nsamples += new_sample._nsamples; + _sum += new_sample._nsamples; + _max = MAX2(_max, new_sample._max); + } +}; + +struct ZStatCounterData { + uint64_t _counter; + + ZStatCounterData() : + _counter(0) {} +}; + +// +// Stat sampler history +// +template +class ZStatSamplerHistoryInterval { +private: + size_t _next; + ZStatSamplerData _samples[size]; + ZStatSamplerData _accumulated; + ZStatSamplerData _total; + +public: + ZStatSamplerHistoryInterval() : + _next(0), + _samples(), + _total(), + _accumulated() {} + + bool add(const ZStatSamplerData& new_sample) { + // Insert sample + const ZStatSamplerData old_sample = _samples[_next]; + _samples[_next] = new_sample; + + // Adjust accumulated + _accumulated._nsamples += new_sample._nsamples; + _accumulated._sum += new_sample._sum; + _accumulated._max = MAX2(_accumulated._max, new_sample._max); + + // Adjust total + _total._nsamples -= old_sample._nsamples; + _total._sum -= old_sample._sum; + _total._nsamples += new_sample._nsamples; + _total._sum += new_sample._sum; + if (_total._max < new_sample._max) { + // Found new max + _total._max = new_sample._max; + } else if (_total._max == old_sample._max) { + // Removed old max, reset and find new max + _total._max = 0; + for (size_t i = 0; i < size; i++) { + if (_total._max < _samples[i]._max) { + _total._max = _samples[i]._max; + } + } + } + + // Adjust next + if (++_next == size) { + _next = 0; + + // Clear accumulated + const ZStatSamplerData zero; + _accumulated = zero; + + // Became full + return true; + } + + // Not yet full + return false; + } + + const ZStatSamplerData& total() const { + return _total; + } + + const ZStatSamplerData& accumulated() const { + return _accumulated; + } +}; + +class ZStatSamplerHistory : public CHeapObj { +private: + ZStatSamplerHistoryInterval<10> _10seconds; + ZStatSamplerHistoryInterval<60> _10minutes; + ZStatSamplerHistoryInterval<60> _10hours; + ZStatSamplerData _total; + + uint64_t avg(uint64_t sum, uint64_t nsamples) const { + return (nsamples > 0) ? sum / nsamples : 0; + } + +public: + ZStatSamplerHistory() : + _10seconds(), + _10minutes(), + _10hours(), + _total() {} + + void add(const ZStatSamplerData& new_sample) { + if (_10seconds.add(new_sample)) { + if (_10minutes.add(_10seconds.total())) { + if (_10hours.add(_10minutes.total())) { + _total.add(_10hours.total()); + } + } + } + } + + uint64_t avg_10_seconds() const { + const uint64_t sum = _10seconds.total()._sum; + const uint64_t nsamples = _10seconds.total()._nsamples; + return avg(sum, nsamples); + } + + uint64_t avg_10_minutes() const { + const uint64_t sum = _10seconds.accumulated()._sum + + _10minutes.total()._sum; + const uint64_t nsamples = _10seconds.accumulated()._nsamples + + _10minutes.total()._nsamples; + return avg(sum, nsamples); + } + + uint64_t avg_10_hours() const { + const uint64_t sum = _10seconds.accumulated()._sum + + _10minutes.accumulated()._sum + + _10hours.total()._sum; + const uint64_t nsamples = _10seconds.accumulated()._nsamples + + _10minutes.accumulated()._nsamples + + _10hours.total()._nsamples; + return avg(sum, nsamples); + } + + uint64_t avg_total() const { + const uint64_t sum = _10seconds.accumulated()._sum + + _10minutes.accumulated()._sum + + _10hours.accumulated()._sum + + _total._sum; + const uint64_t nsamples = _10seconds.accumulated()._nsamples + + _10minutes.accumulated()._nsamples + + _10hours.accumulated()._nsamples + + _total._nsamples; + return avg(sum, nsamples); + } + + uint64_t max_10_seconds() const { + return _10seconds.total()._max; + } + + uint64_t max_10_minutes() const { + return MAX2(_10seconds.accumulated()._max, + _10minutes.total()._max); + } + + uint64_t max_10_hours() const { + return MAX3(_10seconds.accumulated()._max, + _10minutes.accumulated()._max, + _10hours.total()._max); + } + + uint64_t max_total() const { + return MAX4(_10seconds.accumulated()._max, + _10minutes.accumulated()._max, + _10hours.accumulated()._max, + _total._max); + } +}; + +// +// Stat unit printers +// +void ZStatUnitTime(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) { + log.print(" %10s: %-40s " + "%9.3f / %-9.3f " + "%9.3f / %-9.3f " + "%9.3f / %-9.3f " + "%9.3f / %-9.3f ms", + sampler.group(), + sampler.name(), + TimeHelper::counter_to_millis(history.avg_10_seconds()), + TimeHelper::counter_to_millis(history.max_10_seconds()), + TimeHelper::counter_to_millis(history.avg_10_minutes()), + TimeHelper::counter_to_millis(history.max_10_minutes()), + TimeHelper::counter_to_millis(history.avg_10_hours()), + TimeHelper::counter_to_millis(history.max_10_hours()), + TimeHelper::counter_to_millis(history.avg_total()), + TimeHelper::counter_to_millis(history.max_total())); +} + +void ZStatUnitBytes(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) { + log.print(" %10s: %-40s " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " MB", + sampler.group(), + sampler.name(), + history.avg_10_seconds() / M, + history.max_10_seconds() / M, + history.avg_10_minutes() / M, + history.max_10_minutes() / M, + history.avg_10_hours() / M, + history.max_10_hours() / M, + history.avg_total() / M, + history.max_total() / M); +} + +void ZStatUnitThreads(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) { + log.print(" %10s: %-40s " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " threads", + sampler.group(), + sampler.name(), + history.avg_10_seconds(), + history.max_10_seconds(), + history.avg_10_minutes(), + history.max_10_minutes(), + history.avg_10_hours(), + history.max_10_hours(), + history.avg_total(), + history.max_total()); +} + +void ZStatUnitBytesPerSecond(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) { + log.print(" %10s: %-40s " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " MB/s", + sampler.group(), + sampler.name(), + history.avg_10_seconds() / M, + history.max_10_seconds() / M, + history.avg_10_minutes() / M, + history.max_10_minutes() / M, + history.avg_10_hours() / M, + history.max_10_hours() / M, + history.avg_total() / M, + history.max_total() / M); +} + +void ZStatUnitOpsPerSecond(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) { + log.print(" %10s: %-40s " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " " + UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " ops/s", + sampler.group(), + sampler.name(), + history.avg_10_seconds(), + history.max_10_seconds(), + history.avg_10_minutes(), + history.max_10_minutes(), + history.avg_10_hours(), + history.max_10_hours(), + history.avg_total(), + history.max_total()); +} + +// +// Stat value +// +uintptr_t ZStatValue::_base = 0; +uint32_t ZStatValue::_cpu_offset = 0; + +ZStatValue::ZStatValue(const char* group, + const char* name, + uint32_t id, + uint32_t size) : + _group(group), + _name(name), + _id(id), + _offset(_cpu_offset) { + assert(_base == 0, "Already initialized"); + _cpu_offset += size; +} + +template +T* ZStatValue::get_cpu_local(uint32_t cpu) const { + assert(_base != 0, "Not initialized"); + const uintptr_t cpu_base = _base + (_cpu_offset * cpu); + const uintptr_t value_addr = cpu_base + _offset; + return (T*)value_addr; +} + +void ZStatValue::initialize() { + // Finalize and align CPU offset + _cpu_offset = align_up(_cpu_offset, ZCacheLineSize); + + // Allocation aligned memory + const size_t size = _cpu_offset * ZCPU::count(); + _base = ZUtils::alloc_aligned(ZCacheLineSize, size); + memset((void*)_base, 0, size); +} + +const char* ZStatValue::group() const { + return _group; +} + +const char* ZStatValue::name() const { + return _name; +} + +uint32_t ZStatValue::id() const { + return _id; +} + +// +// Stat iterable value +// +template uint32_t ZStatIterableValue::_count = 0; +template T* ZStatIterableValue::_first = NULL; + +template +ZStatIterableValue::ZStatIterableValue(const char* group, + const char* name, + uint32_t size) : + ZStatValue(group, name, _count++, size), + _next(insert()) {} + +template +T* ZStatIterableValue::insert() const { + T** current = &_first; + + while (*current != NULL) { + // First sort by group, then by name + const int group_cmp = strcmp((*current)->group(), group()); + const int name_cmp = strcmp((*current)->name(), name()); + if ((group_cmp > 0) || (group_cmp == 0 && name_cmp > 0)) { + break; + } + + current = &(*current)->_next; + } + + T* const next = *current; + *current = (T*)this; + return next; +} + +// +// Stat sampler +// +ZStatSampler::ZStatSampler(const char* group, const char* name, ZStatUnitPrinter printer) : + ZStatIterableValue(group, name, sizeof(ZStatSamplerData)), + _printer(printer) {} + +ZStatSamplerData* ZStatSampler::get() const { + return get_cpu_local(ZCPU::id()); +} + +ZStatSamplerData ZStatSampler::collect_and_reset() const { + ZStatSamplerData all; + + const uint32_t ncpus = ZCPU::count(); + for (uint32_t i = 0; i < ncpus; i++) { + ZStatSamplerData* const cpu_data = get_cpu_local(i); + if (cpu_data->_nsamples > 0) { + const uint64_t nsamples = Atomic::xchg((uint64_t)0, &cpu_data->_nsamples); + const uint64_t sum = Atomic::xchg((uint64_t)0, &cpu_data->_sum); + const uint64_t max = Atomic::xchg((uint64_t)0, &cpu_data->_max); + all._nsamples += nsamples; + all._sum += sum; + if (all._max < max) { + all._max = max; + } + } + } + + return all; +} + +ZStatUnitPrinter ZStatSampler::printer() const { + return _printer; +} + +// +// Stat counter +// +ZStatCounter::ZStatCounter(const char* group, const char* name, ZStatUnitPrinter printer) : + ZStatIterableValue(group, name, sizeof(ZStatCounterData)), + _sampler(group, name, printer) {} + +ZStatCounterData* ZStatCounter::get() const { + return get_cpu_local(ZCPU::id()); +} + +void ZStatCounter::sample_and_reset() const { + uint64_t counter = 0; + + const uint32_t ncpus = ZCPU::count(); + for (uint32_t i = 0; i < ncpus; i++) { + ZStatCounterData* const cpu_data = get_cpu_local(i); + counter += Atomic::xchg((uint64_t)0, &cpu_data->_counter); + } + + ZStatSample(_sampler, counter); +} + +// +// Stat unsampled counter +// +ZStatUnsampledCounter::ZStatUnsampledCounter(const char* name) : + ZStatIterableValue("Unsampled", name, sizeof(ZStatCounterData)) {} + +ZStatCounterData* ZStatUnsampledCounter::get() const { + return get_cpu_local(ZCPU::id()); +} + +ZStatCounterData ZStatUnsampledCounter::collect_and_reset() const { + ZStatCounterData all; + + const uint32_t ncpus = ZCPU::count(); + for (uint32_t i = 0; i < ncpus; i++) { + ZStatCounterData* const cpu_data = get_cpu_local(i); + all._counter += Atomic::xchg((uint64_t)0, &cpu_data->_counter); + } + + return all; +} + +// +// Stat MMU (Mimimum Mutator Utilization) +// +ZStatMMUPause::ZStatMMUPause() : + _start(0.0), + _end(0.0) {} + +ZStatMMUPause::ZStatMMUPause(const Ticks& start, const Ticks& end) : + _start(TimeHelper::counter_to_millis(start.value())), + _end(TimeHelper::counter_to_millis(end.value())) {} + +double ZStatMMUPause::end() const { + return _end; +} + +double ZStatMMUPause::overlap(double start, double end) const { + const double start_max = MAX2(start, _start); + const double end_min = MIN2(end, _end); + + if (end_min > start_max) { + // Overlap found + return end_min - start_max; + } + + // No overlap + return 0.0; +} + +size_t ZStatMMU::_next = 0; +size_t ZStatMMU::_npauses = 0; +ZStatMMUPause ZStatMMU::_pauses[200]; +double ZStatMMU::_mmu_2ms = 100.0; +double ZStatMMU::_mmu_5ms = 100.0; +double ZStatMMU::_mmu_10ms = 100.0; +double ZStatMMU::_mmu_20ms = 100.0; +double ZStatMMU::_mmu_50ms = 100.0; +double ZStatMMU::_mmu_100ms = 100.0; + +const ZStatMMUPause& ZStatMMU::pause(size_t index) { + return _pauses[(_next - index - 1) % ARRAY_SIZE(_pauses)]; +} + +double ZStatMMU::calculate_mmu(double time_slice) { + const double end = pause(0).end(); + const double start = end - time_slice; + double time_paused = 0.0; + + // Find all overlapping pauses + for (size_t i = 0; i < _npauses; i++) { + const double overlap = pause(i).overlap(start, end); + if (overlap == 0.0) { + // No overlap + break; + } + + time_paused += overlap; + } + + // Calculate MMU + const double time_mutator = time_slice - time_paused; + return percent_of(time_mutator, time_slice); +} + +void ZStatMMU::register_pause(const Ticks& start, const Ticks& end) { + // Add pause + const size_t index = _next++ % ARRAY_SIZE(_pauses); + _pauses[index] = ZStatMMUPause(start, end); + _npauses = MIN2(_npauses + 1, ARRAY_SIZE(_pauses)); + + // Recalculate MMUs + _mmu_2ms = MIN2(_mmu_2ms, calculate_mmu(2)); + _mmu_5ms = MIN2(_mmu_5ms, calculate_mmu(5)); + _mmu_10ms = MIN2(_mmu_10ms, calculate_mmu(10)); + _mmu_20ms = MIN2(_mmu_20ms, calculate_mmu(20)); + _mmu_50ms = MIN2(_mmu_50ms, calculate_mmu(50)); + _mmu_100ms = MIN2(_mmu_100ms, calculate_mmu(100)); +} + +void ZStatMMU::print() { + log_info(gc, mmu)( + "MMU: 2ms/%.1f%%, 5ms/%.1f%%, 10ms/%.1f%%, 20ms/%.1f%%, 50ms/%.1f%%, 100ms/%.1f%%", + _mmu_2ms, _mmu_5ms, _mmu_10ms, _mmu_20ms, _mmu_50ms, _mmu_100ms); +} + +// +// Stat phases +// +ConcurrentGCTimer ZStatPhase::_timer; + +ZStatPhase::ZStatPhase(const char* group, const char* name) : + _sampler(group, name, ZStatUnitTime) {} + +void ZStatPhase::log_start(LogTargetHandle log, bool thread) const { + if (!log.is_enabled()) { + return; + } + + if (thread) { + ResourceMark rm; + log.print("%s (%s)", name(), Thread::current()->name()); + } else { + log.print("%s", name()); + } +} + +void ZStatPhase::log_end(LogTargetHandle log, const Tickspan& duration, bool thread) const { + if (!log.is_enabled()) { + return; + } + + if (thread) { + ResourceMark rm; + log.print("%s (%s) %.3fms", name(), Thread::current()->name(), TimeHelper::counter_to_millis(duration.value())); + } else { + log.print("%s %.3fms", name(), TimeHelper::counter_to_millis(duration.value())); + } +} + +ConcurrentGCTimer* ZStatPhase::timer() { + return &_timer; +} + +const char* ZStatPhase::name() const { + return _sampler.name(); +} + +ZStatPhaseCycle::ZStatPhaseCycle(const char* name) : + ZStatPhase("Collector", name) {} + +void ZStatPhaseCycle::register_start(const Ticks& start) const { + timer()->register_gc_start(start); + + ZTracer::tracer()->report_gc_start(ZCollectedHeap::heap()->gc_cause(), start); + + ZCollectedHeap::heap()->print_heap_before_gc(); + ZCollectedHeap::heap()->trace_heap_before_gc(ZTracer::tracer()); + + log_info(gc, start)("Garbage Collection (%s)", + GCCause::to_string(ZCollectedHeap::heap()->gc_cause())); +} + +#define ZUSED_FMT SIZE_FORMAT "M(%.0lf%%)" +#define ZUSED_ARGS(size, max_capacity) ((size) / M), (percent_of(size, max_capacity)) + +void ZStatPhaseCycle::register_end(const Ticks& start, const Ticks& end) const { + timer()->register_gc_end(end); + + ZCollectedHeap::heap()->print_heap_after_gc(); + ZCollectedHeap::heap()->trace_heap_after_gc(ZTracer::tracer()); + + ZTracer::tracer()->report_gc_end(end, timer()->time_partitions()); + + const Tickspan duration = end - start; + ZStatSample(_sampler, duration.value()); + + ZStatLoad::print(); + ZStatMMU::print(); + ZStatMark::print(); + ZStatRelocation::print(); + ZStatNMethods::print(); + ZStatReferences::print(); + ZStatHeap::print(); + + log_info(gc)("Garbage Collection (%s) " ZUSED_FMT "->" ZUSED_FMT, + GCCause::to_string(ZCollectedHeap::heap()->gc_cause()), + ZUSED_ARGS(ZStatHeap::used_at_mark_start(), ZStatHeap::max_capacity()), + ZUSED_ARGS(ZStatHeap::used_at_relocate_end(), ZStatHeap::max_capacity())); +} + +Tickspan ZStatPhasePause::_max; + +ZStatPhasePause::ZStatPhasePause(const char* name) : + ZStatPhase("Phase", name) {} + +const Tickspan& ZStatPhasePause::max() { + return _max; +} + +void ZStatPhasePause::register_start(const Ticks& start) const { + timer()->register_gc_pause_start(name(), start); + + LogTarget(Debug, gc, phases, start) log; + log_start(log); +} + +void ZStatPhasePause::register_end(const Ticks& start, const Ticks& end) const { + timer()->register_gc_pause_end(end); + + const Tickspan duration = end - start; + ZStatSample(_sampler, duration.value()); + + // Track max pause time + if (_max < duration) { + _max = duration; + } + + // Track minimum mutator utilization + ZStatMMU::register_pause(start, end); + + LogTarget(Info, gc, phases) log; + log_end(log, duration); +} + +ZStatPhaseConcurrent::ZStatPhaseConcurrent(const char* name) : + ZStatPhase("Phase", name) {} + +void ZStatPhaseConcurrent::register_start(const Ticks& start) const { + timer()->register_gc_concurrent_start(name(), start); + + LogTarget(Debug, gc, phases, start) log; + log_start(log); +} + +void ZStatPhaseConcurrent::register_end(const Ticks& start, const Ticks& end) const { + timer()->register_gc_concurrent_end(end); + + const Tickspan duration = end - start; + ZStatSample(_sampler, duration.value()); + + LogTarget(Info, gc, phases) log; + log_end(log, duration); +} + +ZStatSubPhase::ZStatSubPhase(const char* name) : + ZStatPhase("Subphase", name) {} + +void ZStatSubPhase::register_start(const Ticks& start) const { + LogTarget(Debug, gc, phases, start) log; + log_start(log, true /* thread */); +} + +void ZStatSubPhase::register_end(const Ticks& start, const Ticks& end) const { + ZTracer::tracer()->report_thread_phase(*this, start, end); + + const Tickspan duration = end - start; + ZStatSample(_sampler, duration.value()); + + LogTarget(Debug, gc, phases) log; + log_end(log, duration, true /* thread */); +} + +ZStatCriticalPhase::ZStatCriticalPhase(const char* name, bool verbose) : + ZStatPhase("Critical", name), + _counter("Critical", name, ZStatUnitOpsPerSecond), + _verbose(verbose) {} + +void ZStatCriticalPhase::register_start(const Ticks& start) const { + LogTarget(Debug, gc, start) log; + log_start(log, true /* thread */); +} + +void ZStatCriticalPhase::register_end(const Ticks& start, const Ticks& end) const { + ZTracer::tracer()->report_thread_phase(*this, start, end); + + const Tickspan duration = end - start; + ZStatSample(_sampler, duration.value()); + ZStatInc(_counter); + + if (_verbose) { + LogTarget(Info, gc) log; + log_end(log, duration, true /* thread */); + } else { + LogTarget(Debug, gc) log; + log_end(log, duration, true /* thread */); + } +} + +// +// Stat sample/inc +// +void ZStatSample(const ZStatSampler& sampler, uint64_t value, bool trace) { + ZStatSamplerData* const cpu_data = sampler.get(); + Atomic::add(1u, &cpu_data->_nsamples); + Atomic::add(value, &cpu_data->_sum); + + uint64_t max = cpu_data->_max; + for (;;) { + if (max >= value) { + // Not max + break; + } + + const uint64_t new_max = value; + const uint64_t prev_max = Atomic::cmpxchg(new_max, &cpu_data->_max, max); + if (prev_max == max) { + // Success + break; + } + + // Retry + max = prev_max; + } + + if (trace) { + ZTracer::tracer()->report_stat_sampler(sampler, value); + } +} + +void ZStatInc(const ZStatCounter& counter, uint64_t increment, bool trace) { + ZStatCounterData* const cpu_data = counter.get(); + const uint64_t value = Atomic::add(increment, &cpu_data->_counter); + + if (trace) { + ZTracer::tracer()->report_stat_counter(counter, increment, value); + } +} + +void ZStatInc(const ZStatUnsampledCounter& counter, uint64_t increment) { + ZStatCounterData* const cpu_data = counter.get(); + Atomic::add(increment, &cpu_data->_counter); +} + +// +// Stat allocation rate +// +const ZStatUnsampledCounter ZStatAllocRate::_counter("Allocation Rate"); +TruncatedSeq ZStatAllocRate::_rate(ZStatAllocRate::sample_window_sec * ZStatAllocRate::sample_hz); +TruncatedSeq ZStatAllocRate::_rate_avg(ZStatAllocRate::sample_window_sec * ZStatAllocRate::sample_hz); + +const ZStatUnsampledCounter& ZStatAllocRate::counter() { + return _counter; +} + +uint64_t ZStatAllocRate::sample_and_reset() { + const ZStatCounterData bytes_per_sample = _counter.collect_and_reset(); + const uint64_t bytes_per_second = bytes_per_sample._counter * sample_hz; + + _rate.add(bytes_per_second); + _rate_avg.add(_rate.avg()); + + return bytes_per_second; +} + +double ZStatAllocRate::avg() { + return _rate.avg(); +} + +double ZStatAllocRate::avg_sd() { + return _rate_avg.sd(); +} + +// +// Stat thread +// +ZStat::ZStat() : + _metronome(sample_hz) { + set_name("ZStat"); + create_and_start(); +} + +void ZStat::sample_and_collect(ZStatSamplerHistory* history) const { + // Sample counters + for (const ZStatCounter* counter = ZStatCounter::first(); counter != NULL; counter = counter->next()) { + counter->sample_and_reset(); + } + + // Collect samples + for (const ZStatSampler* sampler = ZStatSampler::first(); sampler != NULL; sampler = sampler->next()) { + ZStatSamplerHistory& sampler_history = history[sampler->id()]; + sampler_history.add(sampler->collect_and_reset()); + } +} + +bool ZStat::should_print(LogTargetHandle log) const { + return log.is_enabled() && (_metronome.nticks() % ZStatisticsInterval == 0); +} + +void ZStat::print(LogTargetHandle log, const ZStatSamplerHistory* history) const { + // Print + log.print("=== Garbage Collection Statistics ======================================================================================================================="); + log.print(" Last 10s Last 10m Last 10h Total"); + log.print(" Avg / Max Avg / Max Avg / Max Avg / Max"); + + for (const ZStatSampler* sampler = ZStatSampler::first(); sampler != NULL; sampler = sampler->next()) { + const ZStatSamplerHistory& sampler_history = history[sampler->id()]; + const ZStatUnitPrinter printer = sampler->printer(); + printer(log, *sampler, sampler_history); + } + + log.print("========================================================================================================================================================="); +} + +void ZStat::run_service() { + ZStatSamplerHistory* const history = new ZStatSamplerHistory[ZStatSampler::count()]; + LogTarget(Info, gc, stats) log; + + // Main loop + while (_metronome.wait_for_tick()) { + sample_and_collect(history); + if (should_print(log)) { + print(log, history); + } + } + + delete [] history; +} + +void ZStat::stop_service() { + _metronome.stop(); +} + +// +// Stat table +// +class ZStatTablePrinter { +private: + static const size_t _buffer_size = 256; + + const size_t _column0_width; + const size_t _columnN_width; + char _buffer[_buffer_size]; + +public: + class ZColumn { + private: + char* const _buffer; + const size_t _position; + const size_t _width; + const size_t _width_next; + + ZColumn next() const { + // Insert space between columns + _buffer[_position + _width] = ' '; + return ZColumn(_buffer, _position + _width + 1, _width_next, _width_next); + } + + size_t print(size_t position, const char* fmt, va_list va) { + const int res = jio_vsnprintf(_buffer + position, _buffer_size - position, fmt, va); + if (res < 0) { + return 0; + } + + return (size_t)res; + } + + public: + ZColumn(char* buffer, size_t position, size_t width, size_t width_next) : + _buffer(buffer), + _position(position), + _width(width), + _width_next(width_next) {} + + ZColumn left(const char* fmt, ...) ATTRIBUTE_PRINTF(2, 3) { + va_list va; + + va_start(va, fmt); + const size_t written = print(_position, fmt, va); + va_end(va); + + if (written < _width) { + // Fill empty space + memset(_buffer + _position + written, ' ', _width - written); + } + + return next(); + } + + ZColumn right(const char* fmt, ...) ATTRIBUTE_PRINTF(2, 3) { + va_list va; + + va_start(va, fmt); + const size_t written = print(_position, fmt, va); + va_end(va); + + if (written > _width) { + // Line too long + return fill('?'); + } + + if (written < _width) { + // Short line, move all to right + memmove(_buffer + _position + _width - written, _buffer + _position, written); + + // Fill empty space + memset(_buffer + _position, ' ', _width - written); + } + + return next(); + } + + ZColumn center(const char* fmt, ...) ATTRIBUTE_PRINTF(2, 3) { + va_list va; + + va_start(va, fmt); + const size_t written = print(_position, fmt, va); + va_end(va); + + if (written > _width) { + // Line too long + return fill('?'); + } + + if (written < _width) { + // Short line, move all to center + const size_t start_space = (_width - written) / 2; + const size_t end_space = _width - written - start_space; + memmove(_buffer + _position + start_space, _buffer + _position, written); + + // Fill empty spaces + memset(_buffer + _position, ' ', start_space); + memset(_buffer + _position + start_space + written, ' ', end_space); + } + + return next(); + } + + ZColumn fill(char filler = ' ') { + memset(_buffer + _position, filler, _width); + return next(); + } + + const char* end() { + _buffer[_position] = '\0'; + return _buffer; + } + }; + +public: + ZStatTablePrinter(size_t column0_width, size_t columnN_width) : + _column0_width(column0_width), + _columnN_width(columnN_width) {} + + ZColumn operator()() { + return ZColumn(_buffer, 0, _column0_width, _columnN_width); + } +}; + +// +// Stat cycle +// +uint64_t ZStatCycle::_ncycles = 0; +Ticks ZStatCycle::_start_of_last; +Ticks ZStatCycle::_end_of_last; +NumberSeq ZStatCycle::_normalized_duration(0.3 /* alpha */); + +void ZStatCycle::at_start() { + _start_of_last = Ticks::now(); +} + +void ZStatCycle::at_end(double boost_factor) { + _end_of_last = Ticks::now(); + _ncycles++; + + // Calculate normalized cycle duration. The measured duration is + // normalized using the boost factor to avoid artificial deflation + // of the duration when boost mode is enabled. + const double duration = (_end_of_last - _start_of_last).seconds(); + const double normalized_duration = duration * boost_factor; + _normalized_duration.add(normalized_duration); +} + +uint64_t ZStatCycle::ncycles() { + return _ncycles; +} + +const AbsSeq& ZStatCycle::normalized_duration() { + return _normalized_duration; +} + +double ZStatCycle::time_since_last() { + if (_ncycles == 0) { + // Return time since VM start-up + return os::elapsedTime(); + } + + const Ticks now = Ticks::now(); + const Tickspan time_since_last = now - _end_of_last; + return time_since_last.seconds(); +} + +// +// Stat load +// +void ZStatLoad::print() { + double loadavg[3] = {}; + os::loadavg(loadavg, ARRAY_SIZE(loadavg)); + log_info(gc, load)("Load: %.2f/%.2f/%.2f", loadavg[0], loadavg[1], loadavg[2]); +} + +// +// Stat mark +// +size_t ZStatMark::_nstripes; +size_t ZStatMark::_nproactiveflush; +size_t ZStatMark::_nterminateflush; +size_t ZStatMark::_ntrycomplete; +size_t ZStatMark::_ncontinue; + +void ZStatMark::set_at_mark_start(size_t nstripes) { + _nstripes = nstripes; +} + +void ZStatMark::set_at_mark_end(size_t nproactiveflush, + size_t nterminateflush, + size_t ntrycomplete, + size_t ncontinue) { + _nproactiveflush = nproactiveflush; + _nterminateflush = nterminateflush; + _ntrycomplete = ntrycomplete; + _ncontinue = ncontinue; +} + +void ZStatMark::print() { + log_info(gc, marking)("Mark: " + SIZE_FORMAT " stripe(s), " + SIZE_FORMAT " proactive flush(es), " + SIZE_FORMAT " terminate flush(es), " + SIZE_FORMAT " completion(s), " + SIZE_FORMAT " continuation(s) ", + _nstripes, + _nproactiveflush, + _nterminateflush, + _ntrycomplete, + _ncontinue); +} + +// +// Stat relocation +// +size_t ZStatRelocation::_relocating; +bool ZStatRelocation::_success; + +void ZStatRelocation::set_at_select_relocation_set(size_t relocating) { + _relocating = relocating; +} + +void ZStatRelocation::set_at_relocate_end(bool success) { + _success = success; +} + +void ZStatRelocation::print() { + if (_success) { + log_info(gc, reloc)("Relocation: Successful, " SIZE_FORMAT "M relocated", _relocating / M); + } else { + log_info(gc, reloc)("Relocation: Incomplete"); + } +} + +// +// Stat nmethods +// +void ZStatNMethods::print() { + log_info(gc, nmethod)("NMethods: " SIZE_FORMAT " registered, " SIZE_FORMAT " unregistered", + ZNMethodTable::registered_nmethods(), + ZNMethodTable::unregistered_nmethods()); +} + +// +// Stat references +// +ZStatReferences::ZCount ZStatReferences::_soft; +ZStatReferences::ZCount ZStatReferences::_weak; +ZStatReferences::ZCount ZStatReferences::_final; +ZStatReferences::ZCount ZStatReferences::_phantom; + +void ZStatReferences::set(ZCount* count, size_t encountered, size_t discovered, size_t enqueued) { + count->encountered = encountered; + count->discovered = discovered; + count->enqueued = enqueued; +} + +void ZStatReferences::set_soft(size_t encountered, size_t discovered, size_t enqueued) { + set(&_soft, encountered, discovered, enqueued); +} + +void ZStatReferences::set_weak(size_t encountered, size_t discovered, size_t enqueued) { + set(&_weak, encountered, discovered, enqueued); +} + +void ZStatReferences::set_final(size_t encountered, size_t discovered, size_t enqueued) { + set(&_final, encountered, discovered, enqueued); +} + +void ZStatReferences::set_phantom(size_t encountered, size_t discovered, size_t enqueued) { + set(&_phantom, encountered, discovered, enqueued); +} + +void ZStatReferences::print(const char* name, const ZStatReferences::ZCount& ref) { + log_info(gc, ref)("%s: " + SIZE_FORMAT " encountered, " + SIZE_FORMAT " discovered, " + SIZE_FORMAT " enqueued", + name, + ref.encountered, + ref.discovered, + ref.enqueued); +} + +void ZStatReferences::print() { + print("Soft", _soft); + print("Weak", _weak); + print("Final", _final); + print("Phantom", _phantom); +} + +// +// Stat heap +// +ZStatHeap::ZAtInitialize ZStatHeap::_at_initialize; +ZStatHeap::ZAtMarkStart ZStatHeap::_at_mark_start; +ZStatHeap::ZAtMarkEnd ZStatHeap::_at_mark_end; +ZStatHeap::ZAtRelocateStart ZStatHeap::_at_relocate_start; +ZStatHeap::ZAtRelocateEnd ZStatHeap::_at_relocate_end; + +#define ZSIZE_NA "%9s", "-" +#define ZSIZE_ARGS(size) SIZE_FORMAT_W(8) "M (%.0lf%%)", \ + ((size) / M), (percent_of(size, _at_initialize.max_capacity)) + +size_t ZStatHeap::available(size_t used) { + return _at_initialize.max_capacity - used; +} + +size_t ZStatHeap::reserve(size_t used) { + return MIN2(_at_initialize.max_reserve, available(used)); +} + +size_t ZStatHeap::free(size_t used) { + return available(used) - reserve(used); +} + +void ZStatHeap::set_at_initialize(size_t max_capacity, + size_t max_reserve) { + _at_initialize.max_capacity = max_capacity; + _at_initialize.max_reserve = max_reserve; +} + +void ZStatHeap::set_at_mark_start(size_t capacity, + size_t used) { + _at_mark_start.capacity = capacity; + _at_mark_start.reserve = reserve(used); + _at_mark_start.used = used; + _at_mark_start.free = free(used); +} + +void ZStatHeap::set_at_mark_end(size_t capacity, + size_t allocated, + size_t used) { + _at_mark_end.capacity = capacity; + _at_mark_end.reserve = reserve(used); + _at_mark_end.allocated = allocated; + _at_mark_end.used = used; + _at_mark_end.free = free(used); +} + +void ZStatHeap::set_at_select_relocation_set(size_t live, + size_t garbage, + size_t reclaimed) { + _at_mark_end.live = live; + _at_mark_end.garbage = garbage; + + _at_relocate_start.garbage = garbage - reclaimed; + _at_relocate_start.reclaimed = reclaimed; +} + +void ZStatHeap::set_at_relocate_start(size_t capacity, + size_t allocated, + size_t used) { + _at_relocate_start.capacity = capacity; + _at_relocate_start.reserve = reserve(used); + _at_relocate_start.allocated = allocated; + _at_relocate_start.used = used; + _at_relocate_start.free = free(used); +} + +void ZStatHeap::set_at_relocate_end(size_t capacity, + size_t allocated, + size_t reclaimed, + size_t used, + size_t used_high, + size_t used_low) { + _at_relocate_end.capacity = capacity; + _at_relocate_end.capacity_high = capacity; + _at_relocate_end.capacity_low = _at_mark_start.capacity; + _at_relocate_end.reserve = reserve(used); + _at_relocate_end.reserve_high = reserve(used_high); + _at_relocate_end.reserve_low = reserve(used_low); + _at_relocate_end.garbage = _at_mark_end.garbage - reclaimed; + _at_relocate_end.allocated = allocated; + _at_relocate_end.reclaimed = reclaimed; + _at_relocate_end.used = used; + _at_relocate_end.used_high = used_high; + _at_relocate_end.used_low = used_low; + _at_relocate_end.free = free(used); + _at_relocate_end.free_high = free(used_low); + _at_relocate_end.free_low = free(used_high); +} + +size_t ZStatHeap::max_capacity() { + return _at_initialize.max_capacity; +} + +size_t ZStatHeap::used_at_mark_start() { + return _at_mark_start.used; +} + +size_t ZStatHeap::used_at_relocate_end() { + return _at_relocate_end.used; +} + +void ZStatHeap::print() { + ZStatTablePrinter table(10, 18); + log_info(gc, heap)("%s", table() + .fill() + .center("Mark Start") + .center("Mark End") + .center("Relocate Start") + .center("Relocate End") + .center("High") + .center("Low") + .end()); + log_info(gc, heap)("%s", table() + .right("Capacity:") + .left(ZSIZE_ARGS(_at_mark_start.capacity)) + .left(ZSIZE_ARGS(_at_mark_end.capacity)) + .left(ZSIZE_ARGS(_at_relocate_start.capacity)) + .left(ZSIZE_ARGS(_at_relocate_end.capacity)) + .left(ZSIZE_ARGS(_at_relocate_end.capacity_high)) + .left(ZSIZE_ARGS(_at_relocate_end.capacity_low)) + .end()); + log_info(gc, heap)("%s", table() + .right("Reserve:") + .left(ZSIZE_ARGS(_at_mark_start.reserve)) + .left(ZSIZE_ARGS(_at_mark_end.reserve)) + .left(ZSIZE_ARGS(_at_relocate_start.reserve)) + .left(ZSIZE_ARGS(_at_relocate_end.reserve)) + .left(ZSIZE_ARGS(_at_relocate_end.reserve_high)) + .left(ZSIZE_ARGS(_at_relocate_end.reserve_low)) + .end()); + log_info(gc, heap)("%s", table() + .right("Free:") + .left(ZSIZE_ARGS(_at_mark_start.free)) + .left(ZSIZE_ARGS(_at_mark_end.free)) + .left(ZSIZE_ARGS(_at_relocate_start.free)) + .left(ZSIZE_ARGS(_at_relocate_end.free)) + .left(ZSIZE_ARGS(_at_relocate_end.free_high)) + .left(ZSIZE_ARGS(_at_relocate_end.free_low)) + .end()); + log_info(gc, heap)("%s", table() + .right("Used:") + .left(ZSIZE_ARGS(_at_mark_start.used)) + .left(ZSIZE_ARGS(_at_mark_end.used)) + .left(ZSIZE_ARGS(_at_relocate_start.used)) + .left(ZSIZE_ARGS(_at_relocate_end.used)) + .left(ZSIZE_ARGS(_at_relocate_end.used_high)) + .left(ZSIZE_ARGS(_at_relocate_end.used_low)) + .end()); + log_info(gc, heap)("%s", table() + .right("Live:") + .left(ZSIZE_NA) + .left(ZSIZE_ARGS(_at_mark_end.live)) + .left(ZSIZE_ARGS(_at_mark_end.live /* Same as at mark end */)) + .left(ZSIZE_ARGS(_at_mark_end.live /* Same as at mark end */)) + .left(ZSIZE_NA) + .left(ZSIZE_NA) + .end()); + log_info(gc, heap)("%s", table() + .right("Allocated:") + .left(ZSIZE_NA) + .left(ZSIZE_ARGS(_at_mark_end.allocated)) + .left(ZSIZE_ARGS(_at_relocate_start.allocated)) + .left(ZSIZE_ARGS(_at_relocate_end.allocated)) + .left(ZSIZE_NA) + .left(ZSIZE_NA) + .end()); + log_info(gc, heap)("%s", table() + .right("Garbage:") + .left(ZSIZE_NA) + .left(ZSIZE_ARGS(_at_mark_end.garbage)) + .left(ZSIZE_ARGS(_at_relocate_start.garbage)) + .left(ZSIZE_ARGS(_at_relocate_end.garbage)) + .left(ZSIZE_NA) + .left(ZSIZE_NA) + .end()); + log_info(gc, heap)("%s", table() + .right("Reclaimed:") + .left(ZSIZE_NA) + .left(ZSIZE_NA) + .left(ZSIZE_ARGS(_at_relocate_start.reclaimed)) + .left(ZSIZE_ARGS(_at_relocate_end.reclaimed)) + .left(ZSIZE_NA) + .left(ZSIZE_NA) + .end()); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zStat.hpp 2018-06-01 22:31:07.967061738 +0200 @@ -0,0 +1,519 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZSTAT_HPP +#define SHARE_GC_Z_ZSTAT_HPP + +#include "gc/shared/concurrentGCThread.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/z/zMetronome.hpp" +#include "logging/logHandle.hpp" +#include "memory/allocation.hpp" +#include "utilities/numberSeq.hpp" +#include "utilities/ticks.hpp" + +class ZPage; +class ZStatSampler; +class ZStatSamplerHistory; +struct ZStatCounterData; +struct ZStatSamplerData; + +// +// Stat unit printers +// +typedef void (*ZStatUnitPrinter)(LogTargetHandle log, const ZStatSampler&, const ZStatSamplerHistory&); + +void ZStatUnitTime(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history); +void ZStatUnitBytes(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history); +void ZStatUnitThreads(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history); +void ZStatUnitBytesPerSecond(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history); +void ZStatUnitOpsPerSecond(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history); + +// +// Stat value +// +class ZStatValue { +private: + static uintptr_t _base; + static uint32_t _cpu_offset; + + const char* const _group; + const char* const _name; + const uint32_t _id; + const uint32_t _offset; + +protected: + ZStatValue(const char* group, + const char* name, + uint32_t id, + uint32_t size); + + template T* get_cpu_local(uint32_t cpu) const; + +public: + static void initialize(); + + const char* group() const; + const char* name() const; + uint32_t id() const; +}; + +// +// Stat iterable value +// +template +class ZStatIterableValue : public ZStatValue { +private: + static uint32_t _count; + static T* _first; + + T* _next; + + T* insert() const; + +protected: + ZStatIterableValue(const char* group, + const char* name, + uint32_t size); + +public: + static uint32_t count() { + return _count; + } + + static T* first() { + return _first; + } + + T* next() const { + return _next; + } +}; + +// +// Stat sampler +// +class ZStatSampler : public ZStatIterableValue { +private: + const ZStatUnitPrinter _printer; + +public: + ZStatSampler(const char* group, + const char* name, + ZStatUnitPrinter printer); + + ZStatSamplerData* get() const; + ZStatSamplerData collect_and_reset() const; + + ZStatUnitPrinter printer() const; +}; + +// +// Stat counter +// +class ZStatCounter : public ZStatIterableValue { +private: + const ZStatSampler _sampler; + +public: + ZStatCounter(const char* group, + const char* name, + ZStatUnitPrinter printer); + + ZStatCounterData* get() const; + void sample_and_reset() const; +}; + +// +// Stat unsampled counter +// +class ZStatUnsampledCounter : public ZStatIterableValue { +public: + ZStatUnsampledCounter(const char* name); + + ZStatCounterData* get() const; + ZStatCounterData collect_and_reset() const; +}; + +// +// Stat MMU (Mimimum Mutator Utilization) +// +class ZStatMMUPause { +private: + double _start; + double _end; + +public: + ZStatMMUPause(); + ZStatMMUPause(const Ticks& start, const Ticks& end); + + double end() const; + double overlap(double start, double end) const; +}; + +class ZStatMMU { +private: + static size_t _next; + static size_t _npauses; + static ZStatMMUPause _pauses[200]; // Record the last 200 pauses + + static double _mmu_2ms; + static double _mmu_5ms; + static double _mmu_10ms; + static double _mmu_20ms; + static double _mmu_50ms; + static double _mmu_100ms; + + static const ZStatMMUPause& pause(size_t index); + static double calculate_mmu(double time_slice); + +public: + static void register_pause(const Ticks& start, const Ticks& end); + + static void print(); +}; + +// +// Stat phases +// +class ZStatPhase { +private: + static ConcurrentGCTimer _timer; + +protected: + const ZStatSampler _sampler; + + ZStatPhase(const char* group, const char* name); + + void log_start(LogTargetHandle log, bool thread = false) const; + void log_end(LogTargetHandle log, const Tickspan& duration, bool thread = false) const; + +public: + static ConcurrentGCTimer* timer(); + + const char* name() const; + + virtual void register_start(const Ticks& start) const = 0; + virtual void register_end(const Ticks& start, const Ticks& end) const = 0; +}; + +class ZStatPhaseCycle : public ZStatPhase { +public: + ZStatPhaseCycle(const char* name); + + virtual void register_start(const Ticks& start) const; + virtual void register_end(const Ticks& start, const Ticks& end) const; +}; + +class ZStatPhasePause : public ZStatPhase { +private: + static Tickspan _max; // Max pause time + +public: + ZStatPhasePause(const char* name); + + static const Tickspan& max(); + + virtual void register_start(const Ticks& start) const; + virtual void register_end(const Ticks& start, const Ticks& end) const; +}; + +class ZStatPhaseConcurrent : public ZStatPhase { +public: + ZStatPhaseConcurrent(const char* name); + + virtual void register_start(const Ticks& start) const; + virtual void register_end(const Ticks& start, const Ticks& end) const; +}; + +class ZStatSubPhase : public ZStatPhase { +public: + ZStatSubPhase(const char* name); + + virtual void register_start(const Ticks& start) const; + virtual void register_end(const Ticks& start, const Ticks& end) const; +}; + +class ZStatCriticalPhase : public ZStatPhase { +private: + const ZStatCounter _counter; + const bool _verbose; + +public: + ZStatCriticalPhase(const char* name, bool verbose = true); + + virtual void register_start(const Ticks& start) const; + virtual void register_end(const Ticks& start, const Ticks& end) const; +}; + +// +// Stat timer +// +class ZStatTimer : public StackObj { +private: + const ZStatPhase& _phase; + const Ticks _start; + +public: + ZStatTimer(const ZStatPhase& phase) : + _phase(phase), + _start(Ticks::now()) { + _phase.register_start(_start); + } + + ~ZStatTimer() { + const Ticks end = Ticks::now(); + _phase.register_end(_start, end); + } +}; + +// +// Stat sample/increment +// +void ZStatSample(const ZStatSampler& sampler, uint64_t value, bool trace = ZStatisticsForceTrace); +void ZStatInc(const ZStatCounter& counter, uint64_t increment = 1, bool trace = ZStatisticsForceTrace); +void ZStatInc(const ZStatUnsampledCounter& counter, uint64_t increment = 1); + +// +// Stat allocation rate +// +class ZStatAllocRate : public AllStatic { +private: + static const ZStatUnsampledCounter _counter; + static TruncatedSeq _rate; // B/s + static TruncatedSeq _rate_avg; // B/s + +public: + static const uint64_t sample_window_sec = 1; // seconds + static const uint64_t sample_hz = 10; + + static const ZStatUnsampledCounter& counter(); + static uint64_t sample_and_reset(); + + static double avg(); + static double avg_sd(); +}; + +// +// Stat thread +// +class ZStat : public ConcurrentGCThread { +private: + static const uint64_t sample_hz = 1; + + ZMetronome _metronome; + + void sample_and_collect(ZStatSamplerHistory* history) const; + bool should_print(LogTargetHandle log) const; + void print(LogTargetHandle log, const ZStatSamplerHistory* history) const; + +protected: + virtual void run_service(); + virtual void stop_service(); + +public: + ZStat(); +}; + +// +// Stat cycle +// +class ZStatCycle : public AllStatic { +private: + static uint64_t _ncycles; + static Ticks _start_of_last; + static Ticks _end_of_last; + static NumberSeq _normalized_duration; + +public: + static void at_start(); + static void at_end(double boost_factor); + + static uint64_t ncycles(); + static const AbsSeq& normalized_duration(); + static double time_since_last(); +}; + +// +// Stat load +// +class ZStatLoad : public AllStatic { +public: + static void print(); +}; + +// +// Stat mark +// +class ZStatMark : public AllStatic { +private: + static size_t _nstripes; + static size_t _nproactiveflush; + static size_t _nterminateflush; + static size_t _ntrycomplete; + static size_t _ncontinue; + +public: + static void set_at_mark_start(size_t nstripes); + static void set_at_mark_end(size_t nproactiveflush, + size_t nterminateflush, + size_t ntrycomplete, + size_t ncontinue); + + static void print(); +}; + +// +// Stat relocation +// +class ZStatRelocation : public AllStatic { +private: + static size_t _relocating; + static bool _success; + +public: + static void set_at_select_relocation_set(size_t relocating); + static void set_at_relocate_end(bool success); + + static void print(); +}; + +// +// Stat nmethods +// +class ZStatNMethods : public AllStatic { +public: + static void print(); +}; + +// +// Stat references +// +class ZStatReferences : public AllStatic { +private: + static struct ZCount { + size_t encountered; + size_t discovered; + size_t enqueued; + } _soft, _weak, _final, _phantom; + + static void set(ZCount* count, size_t encountered, size_t discovered, size_t enqueued); + static void print(const char* name, const ZCount& ref); + +public: + static void set_soft(size_t encountered, size_t discovered, size_t enqueued); + static void set_weak(size_t encountered, size_t discovered, size_t enqueued); + static void set_final(size_t encountered, size_t discovered, size_t enqueued); + static void set_phantom(size_t encountered, size_t discovered, size_t enqueued); + + static void print(); +}; + +// +// Stat heap +// +class ZStatHeap : public AllStatic { +private: + static struct ZAtInitialize { + size_t max_capacity; + size_t max_reserve; + } _at_initialize; + + static struct ZAtMarkStart { + size_t capacity; + size_t reserve; + size_t used; + size_t free; + } _at_mark_start; + + static struct ZAtMarkEnd { + size_t capacity; + size_t reserve; + size_t allocated; + size_t used; + size_t free; + size_t live; + size_t garbage; + } _at_mark_end; + + static struct ZAtRelocateStart { + size_t capacity; + size_t reserve; + size_t garbage; + size_t allocated; + size_t reclaimed; + size_t used; + size_t free; + } _at_relocate_start; + + static struct ZAtRelocateEnd { + size_t capacity; + size_t capacity_high; + size_t capacity_low; + size_t reserve; + size_t reserve_high; + size_t reserve_low; + size_t garbage; + size_t allocated; + size_t reclaimed; + size_t used; + size_t used_high; + size_t used_low; + size_t free; + size_t free_high; + size_t free_low; + } _at_relocate_end; + + static size_t available(size_t used); + static size_t reserve(size_t used); + static size_t free(size_t used); + +public: + static void set_at_initialize(size_t max_capacity, + size_t max_reserve); + static void set_at_mark_start(size_t capacity, + size_t used); + static void set_at_mark_end(size_t capacity, + size_t allocated, + size_t used); + static void set_at_select_relocation_set(size_t live, + size_t garbage, + size_t reclaimed); + static void set_at_relocate_start(size_t capacity, + size_t allocated, + size_t used); + static void set_at_relocate_end(size_t capacity, + size_t allocated, + size_t reclaimed, + size_t used, + size_t used_high, + size_t used_low); + + static size_t max_capacity(); + static size_t used_at_mark_start(); + static size_t used_at_relocate_end(); + + static void print(); +}; + +#endif // SHARE_GC_Z_ZSTAT_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zTask.cpp 2018-06-01 22:31:08.358078616 +0200 @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zThread.hpp" + +ZTask::GangTask::GangTask(ZTask* ztask, const char* name) : + AbstractGangTask(name), + _ztask(ztask) {} + +void ZTask::GangTask::work(uint worker_id) { + ZThread::set_worker_id(worker_id); + _ztask->work(); + ZThread::clear_worker_id(); +} + +ZTask::ZTask(const char* name) : + _gang_task(this, name) {} + +const char* ZTask::name() const { + return _gang_task.name(); +} + +AbstractGangTask* ZTask::gang_task() { + return &_gang_task; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zTask.hpp 2018-06-01 22:31:08.749095494 +0200 @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZTASK_HPP +#define SHARE_GC_Z_ZTASK_HPP + +#include "gc/shared/workgroup.hpp" +#include "memory/allocation.hpp" + +class ZTask : public StackObj { +private: + class GangTask : public AbstractGangTask { + private: + ZTask* const _ztask; + + public: + GangTask(ZTask* ztask, const char* name); + + virtual void work(uint worker_id); + }; + + GangTask _gang_task; + +public: + ZTask(const char* name); + + const char* name() const; + AbstractGangTask* gang_task(); + + virtual void work() = 0; +}; + +#endif // SHARE_GC_Z_ZTASK_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zThread.cpp 2018-06-01 22:31:09.137112242 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zThread.hpp" +#include "runtime/thread.hpp" +#include "utilities/debug.hpp" + +__thread bool ZThread::_initialized; +__thread uintptr_t ZThread::_id; +__thread bool ZThread::_is_vm; +__thread bool ZThread::_is_java; +__thread bool ZThread::_is_worker; +__thread uint ZThread::_worker_id; + +void ZThread::initialize() { + assert(!_initialized, "Already initialized"); + const Thread* const thread = Thread::current(); + _initialized = true; + _id = (uintptr_t)thread; + _is_vm = thread->is_VM_thread(); + _is_java = thread->is_Java_thread(); + _is_worker = thread->is_Worker_thread(); + _worker_id = (uint)-1; +} + +const char* ZThread::name() { + const Thread* const thread = Thread::current(); + if (thread->is_Named_thread()) { + const NamedThread* const named = (const NamedThread*)thread; + return named->name(); + } else if (thread->is_Java_thread()) { + return "Java"; + } + + return "Unknown"; +} + +bool ZThread::has_worker_id() { + return _initialized && + _is_worker && + _worker_id != (uint)-1; +} + +void ZThread::set_worker_id(uint worker_id) { + ensure_initialized(); + assert(!has_worker_id(), "Worker id already initialized"); + _worker_id = worker_id; +} + +void ZThread::clear_worker_id() { + assert(has_worker_id(), "Worker id not initialized"); + _worker_id = (uint)-1; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zThread.hpp 2018-06-01 22:31:09.510128343 +0200 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZTHREAD_HPP +#define SHARE_GC_Z_ZTHREAD_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +class ZThread : public AllStatic { + friend class ZTask; + +private: + static __thread bool _initialized; + static __thread uintptr_t _id; + static __thread bool _is_vm; + static __thread bool _is_java; + static __thread bool _is_worker; + static __thread uint _worker_id; + + static void initialize(); + + static void ensure_initialized() { + if (!_initialized) { + initialize(); + } + } + + static bool has_worker_id(); + static void set_worker_id(uint worker_id); + static void clear_worker_id(); + +public: + static const char* name(); + + static uintptr_t id() { + ensure_initialized(); + return _id; + } + + static bool is_vm() { + ensure_initialized(); + return _is_vm; + } + + static bool is_java() { + ensure_initialized(); + return _is_java; + } + + static bool is_worker() { + ensure_initialized(); + return _is_worker; + } + + static uint worker_id() { + assert(has_worker_id(), "Worker id not initialized"); + return _worker_id; + } +}; + +#endif // SHARE_GC_Z_ZTHREAD_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zThreadLocalData.hpp 2018-06-01 22:31:09.879144271 +0200 @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZTHREADLOCALDATA_HPP +#define SHARE_GC_Z_ZTHREADLOCALDATA_HPP + +#include "gc/z/zMarkStack.hpp" +#include "runtime/thread.hpp" +#include "utilities/debug.hpp" +#include "utilities/sizes.hpp" + +class ZThreadLocalData { +private: + uintptr_t _address_bad_mask; + ZMarkThreadLocalStacks _stacks; + + ZThreadLocalData() : + _address_bad_mask(0), + _stacks() {} + + static ZThreadLocalData* data(Thread* thread) { + return thread->gc_data(); + } + +public: + static void create(Thread* thread) { + new (data(thread)) ZThreadLocalData(); + } + + static void destroy(Thread* thread) { + data(thread)->~ZThreadLocalData(); + } + + static void set_address_bad_mask(Thread* thread, uintptr_t mask) { + data(thread)->_address_bad_mask = mask; + } + + static ZMarkThreadLocalStacks* stacks(Thread* thread) { + return &data(thread)->_stacks; + } + + static ByteSize address_bad_mask_offset() { + return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _address_bad_mask); + } +}; + +#endif // SHARE_GC_Z_ZTHREADLOCALDATA_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zTracer.cpp 2018-06-01 22:31:10.249160243 +0200 @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTracer.hpp" +#include "gc/shared/gcId.hpp" +#include "gc/shared/gcLocker.hpp" +#include "jfr/jfrEvents.hpp" +#include "runtime/safepointVerifiers.hpp" + +ZTracer* ZTracer::_tracer = NULL; + +ZTracer::ZTracer() : + GCTracer(Z) {} + +void ZTracer::initialize() { + _tracer = new (ResourceObj::C_HEAP, mtGC) ZTracer(); +} + +void ZTracer::send_stat_counter(uint32_t counter_id, uint64_t increment, uint64_t value) { + NoSafepointVerifier nsv(true, !SafepointSynchronize::is_at_safepoint()); + + EventZStatisticsCounter e; + if (e.should_commit()) { + e.set_id(counter_id); + e.set_increment(increment); + e.set_value(value); + e.commit(); + } +} + +void ZTracer::send_stat_sampler(uint32_t sampler_id, uint64_t value) { + NoSafepointVerifier nsv(true, !SafepointSynchronize::is_at_safepoint()); + + EventZStatisticsSampler e; + if (e.should_commit()) { + e.set_id(sampler_id); + e.set_value(value); + e.commit(); + } +} + +void ZTracer::send_thread_phase(const char* name, const Ticks& start, const Ticks& end) { + NoSafepointVerifier nsv(true, !SafepointSynchronize::is_at_safepoint()); + + EventZThreadPhase e(UNTIMED); + if (e.should_commit()) { + e.set_gcId(GCId::current_or_undefined()); + e.set_name(name); + e.set_starttime(start); + e.set_endtime(end); + e.commit(); + } +} + +void ZTracer::send_page_alloc(size_t size, size_t used, size_t free, size_t cache, bool nonblocking, bool noreserve) { + NoSafepointVerifier nsv(true, !SafepointSynchronize::is_at_safepoint()); + + EventZPageAllocation e; + if (e.should_commit()) { + e.set_pageSize(size); + e.set_usedAfter(used); + e.set_freeAfter(free); + e.set_inCacheAfter(cache); + e.set_nonBlocking(nonblocking); + e.set_noReserve(noreserve); + e.commit(); + } +} + +void ZTracer::report_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value) { + send_stat_counter(counter.id(), increment, value); +} + +void ZTracer::report_stat_sampler(const ZStatSampler& sampler, uint64_t value) { + send_stat_sampler(sampler.id(), value); +} + +void ZTracer::report_thread_phase(const ZStatPhase& phase, const Ticks& start, const Ticks& end) { + send_thread_phase(phase.name(), start, end); +} + +void ZTracer::report_thread_phase(const char* name, const Ticks& start, const Ticks& end) { + send_thread_phase(name, start, end); +} + +void ZTracer::report_page_alloc(size_t size, size_t used, size_t free, size_t cache, ZAllocationFlags flags) { + send_page_alloc(size, used, free, cache, flags.non_blocking(), flags.no_reserve()); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zTracer.hpp 2018-06-01 22:31:10.625176473 +0200 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZTRACER_HPP +#define SHARE_GC_Z_ZTRACER_HPP + +#include "gc/shared/gcTrace.hpp" +#include "gc/z/zAllocationFlags.hpp" + +class ZStatCounter; +class ZStatPhase; +class ZStatSampler; + +class ZTracer : public GCTracer { +private: + static ZTracer* _tracer; + + ZTracer(); + + void send_stat_counter(uint32_t counter_id, uint64_t increment, uint64_t value); + void send_stat_sampler(uint32_t sampler_id, uint64_t value); + void send_thread_phase(const char* name, const Ticks& start, const Ticks& end); + void send_page_alloc(size_t size, size_t used, size_t free, size_t cache, bool nonblocking, bool noreserve); + +public: + static ZTracer* tracer(); + static void initialize(); + + void report_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value); + void report_stat_sampler(const ZStatSampler& sampler, uint64_t value); + void report_thread_phase(const ZStatPhase& phase, const Ticks& start, const Ticks& end); + void report_thread_phase(const char* name, const Ticks& start, const Ticks& end); + void report_page_alloc(size_t size, size_t used, size_t free, size_t cache, ZAllocationFlags flags); +}; + +class ZTraceThreadPhase : public StackObj { +private: + const Ticks _start; + const char* const _name; + +public: + ZTraceThreadPhase(const char* name); + ~ZTraceThreadPhase(); +}; + +#endif // SHARE_GC_Z_ZTRACER_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zTracer.inline.hpp 2018-06-01 22:31:10.999192617 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZTRACER_INLINE_HPP +#define SHARE_GC_Z_ZTRACER_INLINE_HPP + +#include "gc/z/zTracer.hpp" + +inline ZTracer* ZTracer::tracer() { + return _tracer; +} + +inline ZTraceThreadPhase::ZTraceThreadPhase(const char* name) : + _start(Ticks::now()), + _name(name) {} + +inline ZTraceThreadPhase::~ZTraceThreadPhase() { + ZTracer::tracer()->report_thread_phase(_name, _start, Ticks::now()); +} + +#endif // SHARE_GC_Z_ZTRACER_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zUtils.cpp 2018-06-01 22:31:11.366208459 +0200 @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "utilities/debug.hpp" + +#include + +uintptr_t ZUtils::alloc_aligned(size_t alignment, size_t size) { + void* res = NULL; + + if (posix_memalign(&res, alignment, size) != 0) { + fatal("posix_memalign() failed"); + } + + memset(res, 0, size); + + return (uintptr_t)res; +} + +void ZUtils::insert_filler_object(uintptr_t addr, size_t size) { + const size_t fill_size_in_words = bytes_to_words(size); + if (fill_size_in_words >= CollectedHeap::min_fill_size()) { + CollectedHeap::fill_with_objects((HeapWord*)ZAddress::good(addr), fill_size_in_words); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zUtils.hpp 2018-06-01 22:31:11.739224560 +0200 @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZUTILS_HPP +#define SHARE_GC_Z_ZUTILS_HPP + +#include "memory/allocation.hpp" + +class ZUtils : public AllStatic { +public: + // Allocation + static uintptr_t alloc_aligned(size_t alignment, size_t size); + + // Power of two + static size_t round_up_power_of_2(size_t value); + static size_t round_down_power_of_2(size_t value); + + // Size convertion + static size_t bytes_to_words(size_t size_in_words); + static size_t words_to_bytes(size_t size_in_words); + + // Object + static size_t object_size(uintptr_t addr); + static void object_copy(uintptr_t from, uintptr_t to, size_t size); + + // Filler + static void insert_filler_object(uintptr_t addr, size_t size); +}; + +#endif // SHARE_GC_Z_ZUTILS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zUtils.inline.hpp 2018-06-01 22:31:12.113240704 +0200 @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZUTILS_INLINE_HPP +#define SHARE_GC_Z_ZUTILS_INLINE_HPP + +#include "gc/z/zOop.inline.hpp" +#include "gc/z/zUtils.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/align.hpp" +#include "utilities/copy.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +inline size_t ZUtils::round_up_power_of_2(size_t value) { + assert(value != 0, "Invalid value"); + + if (is_power_of_2(value)) { + return value; + } + + return (size_t)1 << (log2_intptr(value) + 1); +} + +inline size_t ZUtils::round_down_power_of_2(size_t value) { + assert(value != 0, "Invalid value"); + return (size_t)1 << log2_intptr(value); +} + +inline size_t ZUtils::bytes_to_words(size_t size_in_bytes) { + assert(is_aligned(size_in_bytes, BytesPerWord), "Size not word aligned"); + return size_in_bytes >> LogBytesPerWord; +} + +inline size_t ZUtils::words_to_bytes(size_t size_in_words) { + return size_in_words << LogBytesPerWord; +} + +inline size_t ZUtils::object_size(uintptr_t addr) { + return words_to_bytes(ZOop::to_oop(addr)->size()); +} + +inline void ZUtils::object_copy(uintptr_t from, uintptr_t to, size_t size) { + Copy::aligned_disjoint_words((HeapWord*)from, (HeapWord*)to, bytes_to_words(size)); +} + +#endif // SHARE_GC_Z_ZUTILS_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zValue.hpp 2018-06-01 22:31:12.491257021 +0200 @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZVALUE_HPP +#define SHARE_GC_Z_ZVALUE_HPP + +#include "memory/allocation.hpp" +#include "gc/z/zCPU.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zNUMA.hpp" +#include "gc/z/zThread.hpp" +#include "gc/z/zUtils.hpp" +#include "utilities/align.hpp" + +template +class ZValueStorage : public AllStatic { +private: + static uintptr_t _top; + static uintptr_t _end; + +public: + static const size_t offset = 4 * K; + + static uintptr_t alloc(size_t size) { + guarantee(size <= offset, "Allocation too large"); + + // Allocate entry in existing memory block + const uintptr_t addr = align_up(_top, S::alignment()); + _top = addr + size; + + if (_top < _end) { + // Success + return addr; + } + + // Allocate new block of memory + const size_t block_alignment = offset; + const size_t block_size = offset * S::count(); + _top = ZUtils::alloc_aligned(block_alignment, block_size); + _end = _top + offset; + + // Retry allocation + return alloc(size); + } +}; + +template uintptr_t ZValueStorage::_end = 0; +template uintptr_t ZValueStorage::_top = 0; + +class ZContendedStorage : public ZValueStorage { +public: + static size_t alignment() { + return ZCacheLineSize; + } + + static uint32_t count() { + return 1; + } + + static uint32_t id() { + return 0; + } +}; + +class ZPerCPUStorage : public ZValueStorage { +public: + static size_t alignment() { + return sizeof(uintptr_t); + } + + static uint32_t count() { + return ZCPU::count(); + } + + static uint32_t id() { + return ZCPU::id(); + } +}; + +class ZPerNUMAStorage : public ZValueStorage { +public: + static size_t alignment() { + return sizeof(uintptr_t); + } + + static uint32_t count() { + return ZNUMA::count(); + } + + static uint32_t id() { + return ZNUMA::id(); + } +}; + +class ZPerWorkerStorage : public ZValueStorage { +public: + static size_t alignment() { + return sizeof(uintptr_t); + } + + static uint32_t count() { + return MAX2(ParallelGCThreads, ConcGCThreads); + } + + static uint32_t id() { + return ZThread::worker_id(); + } +}; + +template +class ZValueIterator; + +template +class ZValue { +private: + const uintptr_t _addr; + + uintptr_t value_addr(uint32_t value_id) const { + return _addr + (value_id * S::offset); + } + +public: + ZValue() : + _addr(S::alloc(sizeof(T))) { + // Initialize all instances + ZValueIterator iter(this); + for (T* addr; iter.next(&addr);) { + ::new (addr) T; + } + } + + ZValue(const T& value) : + _addr(S::alloc(sizeof(T))) { + // Initialize all instances + ZValueIterator iter(this); + for (T* addr; iter.next(&addr);) { + ::new (addr) T(value); + } + } + + // Not implemented + ZValue(const ZValue& value); + ZValue& operator=(const ZValue& value); + + const T* addr(uint32_t value_id = S::id()) const { + return reinterpret_cast(value_addr(value_id)); + } + + T* addr(uint32_t value_id = S::id()) { + return reinterpret_cast(value_addr(value_id)); + } + + const T& get(uint32_t value_id = S::id()) const { + return *addr(value_id); + } + + T& get(uint32_t value_id = S::id()) { + return *addr(value_id); + } + + void set(const T& value, uint32_t value_id = S::id()) { + get(value_id) = value; + } + + void set_all(const T& value) { + ZValueIterator iter(this); + for (T* addr; iter.next(&addr);) { + *addr = value; + } + } +}; + +template +class ZContended : public ZValue { +public: + ZContended() : + ZValue() {} + + ZContended(const T& value) : + ZValue(value) {} + + using ZValue::operator=; +}; + +template +class ZPerCPU : public ZValue { +public: + ZPerCPU() : + ZValue() {} + + ZPerCPU(const T& value) : + ZValue(value) {} + + using ZValue::operator=; +}; + +template +class ZPerNUMA : public ZValue { +public: + ZPerNUMA() : + ZValue() {} + + ZPerNUMA(const T& value) : + ZValue(value) {} + + using ZValue::operator=; +}; + +template +class ZPerWorker : public ZValue { +public: + ZPerWorker() : + ZValue() {} + + ZPerWorker(const T& value) : + ZValue(value) {} + + using ZValue::operator=; +}; + +template +class ZValueIterator { +private: + ZValue* const _value; + uint32_t _value_id; + +public: + ZValueIterator(ZValue* value) : + _value(value), + _value_id(0) {} + + bool next(T** value) { + if (_value_id < S::count()) { + *value = _value->addr(_value_id++); + return true; + } + return false; + } +}; + +template +class ZPerCPUIterator : public ZValueIterator { +public: + ZPerCPUIterator(ZPerCPU* value) : + ZValueIterator(value) {} +}; + +template +class ZPerNUMAIterator : public ZValueIterator { +public: + ZPerNUMAIterator(ZPerNUMA* value) : + ZValueIterator(value) {} +}; + +template +class ZPerWorkerIterator : public ZValueIterator { +public: + ZPerWorkerIterator(ZPerWorker* value) : + ZValueIterator(value) {} +}; + +template +class ZValueConstIterator { +private: + const ZValue* const _value; + uint32_t _value_id; + +public: + ZValueConstIterator(const ZValue* value) : + _value(value), + _value_id(0) {} + + bool next(const T** value) { + if (_value_id < S::count()) { + *value = _value->addr(_value_id++); + return true; + } + return false; + } +}; + +template +class ZPerCPUConstIterator : public ZValueConstIterator { +public: + ZPerCPUConstIterator(const ZPerCPU* value) : + ZValueConstIterator(value) {} +}; + +template +class ZPerNUMAConstIterator : public ZValueConstIterator { +public: + ZPerNUMAConstIterator(const ZPerNUMA* value) : + ZValueConstIterator(value) {} +}; + +template +class ZPerWorkerConstIterator : public ZValueConstIterator { +public: + ZPerWorkerConstIterator(const ZPerWorker* value) : + ZValueConstIterator(value) {} +}; + +#endif // SHARE_GC_Z_ZVALUE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zVirtualMemory.cpp 2018-06-01 22:31:12.868273294 +0200 @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zVirtualMemory.inline.hpp" +#include "services/memTracker.hpp" + +ZVirtualMemoryManager::ZVirtualMemoryManager() : + _manager(), + _initialized(false) { + // Reserve address space + if (!reserve(ZAddressSpaceStart, ZAddressSpaceSize)) { + return; + } + + // Make the complete address view free + _manager.free(0, ZAddressOffsetMax); + + // Register address space with native memory tracker + nmt_reserve(ZAddressSpaceStart, ZAddressSpaceSize); + + // Successfully initialized + _initialized = true; +} + +void ZVirtualMemoryManager::nmt_reserve(uintptr_t start, size_t size) { + MemTracker::record_virtual_memory_reserve((void*)start, size, CALLER_PC); + MemTracker::record_virtual_memory_type((void*)start, mtJavaHeap); +} + +bool ZVirtualMemoryManager::is_initialized() const { + return _initialized; +} + +ZVirtualMemory ZVirtualMemoryManager::alloc(size_t size, bool alloc_from_front) { + uintptr_t start; + + if (alloc_from_front || size <= ZPageSizeSmall) { + // Small page + start = _manager.alloc_from_front(size); + } else { + // Medium/Large page + start = _manager.alloc_from_back(size); + } + + return ZVirtualMemory(start, size); +} + +void ZVirtualMemoryManager::free(ZVirtualMemory vmem) { + _manager.free(vmem.start(), vmem.size()); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zVirtualMemory.hpp 2018-06-01 22:31:13.245289568 +0200 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZVIRTUALMEMORY_HPP +#define SHARE_GC_Z_ZVIRTUALMEMORY_HPP + +#include "gc/z/zMemory.hpp" +#include "memory/allocation.hpp" + +class ZVirtualMemory { + friend class VMStructs; + +private: + uintptr_t _start; + uintptr_t _end; + +public: + ZVirtualMemory(); + ZVirtualMemory(uintptr_t start, size_t size); + + bool is_null() const; + uintptr_t start() const; + uintptr_t end() const; + size_t size() const; + ZVirtualMemory split(size_t size); + void clear(); +}; + +class ZVirtualMemoryManager { +private: + ZMemoryManager _manager; + bool _initialized; + + bool reserve(uintptr_t start, size_t size); + void nmt_reserve(uintptr_t start, size_t size); + +public: + ZVirtualMemoryManager(); + + bool is_initialized() const; + + ZVirtualMemory alloc(size_t size, bool alloc_from_front = false); + void free(ZVirtualMemory vmem); +}; + +#endif // SHARE_GC_Z_ZVIRTUALMEMORY_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp 2018-06-01 22:31:13.616305583 +0200 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZVIRTUALMEMORY_INLINE_HPP +#define SHARE_GC_Z_ZVIRTUALMEMORY_INLINE_HPP + +#include "gc/z/zMemory.inline.hpp" +#include "gc/z/zVirtualMemory.hpp" + +inline ZVirtualMemory::ZVirtualMemory() : + _start(UINTPTR_MAX), + _end(UINTPTR_MAX) {} + +inline ZVirtualMemory::ZVirtualMemory(uintptr_t start, size_t size) : + _start(start), + _end(start + size) {} + +inline bool ZVirtualMemory::is_null() const { + return _start == UINTPTR_MAX; +} + +inline uintptr_t ZVirtualMemory::start() const { + return _start; +} + +inline uintptr_t ZVirtualMemory::end() const { + return _end; +} + +inline size_t ZVirtualMemory::size() const { + return _end - _start; +} + +inline ZVirtualMemory ZVirtualMemory::split(size_t split_size) { + assert(split_size <= size(), "precondition"); + ZVirtualMemory mem(_start, split_size); + _start += split_size; + return mem; +} + +inline void ZVirtualMemory::clear() { + _start = UINTPTR_MAX; + _end = UINTPTR_MAX; +} + +#endif // SHARE_GC_Z_ZVIRTUALMEMORY_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zWeakRootsProcessor.cpp 2018-06-01 22:31:13.993321856 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zHeap.inline.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zStat.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zThread.hpp" +#include "runtime/jniHandles.hpp" + +ZWeakRootsProcessor::ZWeakRootsProcessor(ZWorkers* workers) : + _workers(workers) {} + +class ZProcessWeakRootsTask : public ZTask { +private: + ZWeakRootsIterator _weak_roots; + +public: + ZProcessWeakRootsTask() : + ZTask("ZProcessWeakRootsTask"), + _weak_roots() {} + + virtual void work() { + ZPhantomIsAliveObjectClosure is_alive; + ZPhantomKeepAliveOopClosure keep_alive; + _weak_roots.weak_oops_do(&is_alive, &keep_alive); + } +}; + +void ZWeakRootsProcessor::process_weak_roots() { + ZProcessWeakRootsTask task; + _workers->run_parallel(&task); +} + +class ZProcessConcurrentWeakRootsTask : public ZTask { +private: + ZConcurrentWeakRootsIterator _concurrent_weak_roots; + +public: + ZProcessConcurrentWeakRootsTask() : + ZTask("ZProcessConccurentWeakRootsTask"), + _concurrent_weak_roots() {} + + virtual void work() { + ZPhantomCleanOopClosure cl; + _concurrent_weak_roots.oops_do(&cl); + } +}; + +void ZWeakRootsProcessor::process_concurrent_weak_roots() { + ZProcessConcurrentWeakRootsTask task; + _workers->run_concurrent(&task); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zWeakRootsProcessor.hpp 2018-06-01 22:31:14.377338432 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZWEAKROOTSPROCESSOR_HPP +#define SHARE_GC_Z_ZWEAKROOTSPROCESSOR_HPP + +#include "gc/z/zValue.hpp" + +class ZWorkers; + +class ZWeakRootsProcessor { +private: + ZWorkers* const _workers; + +public: + ZWeakRootsProcessor(ZWorkers* workers); + + void process_weak_roots(); + void process_concurrent_weak_roots(); +}; + +#endif // SHARE_GC_Z_ZWEAKROOTSPROCESSOR_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zWorkers.cpp 2018-06-01 22:31:14.762355051 +0200 @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zWorkers.inline.hpp" +#include "runtime/os.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/safepoint.hpp" + +uint ZWorkers::calculate_ncpus(double share_in_percent) { + return ceil(os::initial_active_processor_count() * share_in_percent / 100.0); +} + +uint ZWorkers::calculate_nparallel() { + // Use 60% of the CPUs, rounded up. We would like to use as many threads as + // possible to increase parallelism. However, using a thread count that is + // close to the number of processors tends to lead to over-provisioning and + // scheduling latency issues. Using 60% of the active processors appears to + // be a fairly good balance. + return calculate_ncpus(60.0); +} + +uint ZWorkers::calculate_nconcurrent() { + // Use 12.5% of the CPUs, rounded up. The number of concurrent threads we + // would like to use heavily depends on the type of workload we are running. + // Using too many threads will have a nagative impact on the application + // throughput, while using too few threads will prolong the GC-cycle and + // we then risk being out-run by the application. Using 12.5% of the active + // processors appears to be a fairly good balance. + return calculate_ncpus(12.5); +} + +class ZWorkersWarmupTask : public ZTask { +private: + const uint _nworkers; + uint _started; + Monitor _monitor; + +public: + ZWorkersWarmupTask(uint nworkers) : + ZTask("ZWorkersWarmupTask"), + _nworkers(nworkers), + _started(0), + _monitor(Monitor::leaf, "ZWorkersWarmup", false, Monitor::_safepoint_check_never) {} + + virtual void work() { + // Wait for all threads to start + MonitorLockerEx ml(&_monitor, Monitor::_no_safepoint_check_flag); + if (++_started == _nworkers) { + // All threads started + ml.notify_all(); + } else { + while (_started != _nworkers) { + ml.wait(Monitor::_no_safepoint_check_flag); + } + } + } +}; + +ZWorkers::ZWorkers() : + _boost(false), + _workers("ZWorker", + nworkers(), + true /* are_GC_task_threads */, + true /* are_ConcurrentGC_threads */) { + + log_info(gc, init)("Workers: %u parallel, %u concurrent", nparallel(), nconcurrent()); + + // Initialize worker threads + _workers.initialize_workers(); + + // Warm up worker threads by having them execute a dummy task. + // This helps reduce latency in early GC pauses, which otherwise + // would have to take on any warmup costs. + ZWorkersWarmupTask task(nworkers()); + run(&task, nworkers()); +} + +void ZWorkers::set_boost(bool boost) { + if (boost) { + log_debug(gc)("Boosting workers"); + } + + _boost = boost; +} + +void ZWorkers::run(ZTask* task, uint nworkers) { + log_debug(gc, task)("Executing Task: %s, Active Workers: %u", task->name(), nworkers); + _workers.update_active_workers(nworkers); + _workers.run_task(task->gang_task()); +} + +void ZWorkers::run_parallel(ZTask* task) { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at a safepoint"); + run(task, nparallel()); +} + +void ZWorkers::run_concurrent(ZTask* task) { + run(task, nconcurrent()); +} + +void ZWorkers::threads_do(ThreadClosure* tc) const { + _workers.threads_do(tc); +} + +void ZWorkers::print_threads_on(outputStream* st) const { + _workers.print_worker_threads_on(st); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zWorkers.hpp 2018-06-01 22:31:15.145371583 +0200 @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZWORKERS_HPP +#define SHARE_GC_Z_ZWORKERS_HPP + +#include "gc/shared/workgroup.hpp" +#include "memory/allocation.hpp" + +class ZTask; + +class ZWorkers { +private: + bool _boost; + WorkGang _workers; + + static uint calculate_ncpus(double share_in_percent); + + void run(ZTask* task, uint nworkers); + +public: + static uint calculate_nparallel(); + static uint calculate_nconcurrent(); + + ZWorkers(); + + uint nparallel() const; + uint nparallel_no_boost() const; + uint nconcurrent() const; + uint nconcurrent_no_boost() const; + uint nworkers() const; + + void set_boost(bool boost); + + void run_parallel(ZTask* task); + void run_concurrent(ZTask* task); + + void threads_do(ThreadClosure* tc) const; + void print_threads_on(outputStream* st) const; +}; + +#endif // SHARE_GC_Z_ZWORKERS_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zWorkers.inline.hpp 2018-06-01 22:31:15.523387900 +0200 @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZWORKERS_INLINE_HPP +#define SHARE_GC_Z_ZWORKERS_INLINE_HPP + +#include "gc/z/zWorkers.hpp" +#include "utilities/globalDefinitions.hpp" + +inline uint ZWorkers::nparallel() const { + return _boost ? nworkers() : nparallel_no_boost(); +} + +inline uint ZWorkers::nparallel_no_boost() const { + return ParallelGCThreads; +} + +inline uint ZWorkers::nconcurrent() const { + return _boost ? nworkers() : nconcurrent_no_boost(); +} + +inline uint ZWorkers::nconcurrent_no_boost() const { + return ConcGCThreads; +} + +inline uint ZWorkers::nworkers() const { + return MAX2(ParallelGCThreads, ConcGCThreads); +} + +#endif // SHARE_GC_Z_ZWORKERS_INLINE_HPP --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/java.base/share/legal/c-libutl.md 2018-06-01 22:31:15.899404131 +0200 @@ -0,0 +1,35 @@ +## c-libutl 20160225 + +### c-libutl License +``` + +This software is distributed under the terms of the BSD license. + +== BSD LICENSE =============================================================== + + (C) 2009 by Remo Dentato (rdentato@gmail.com) + + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +http://opensource.org/licenses/bsd-license.php + +``` --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java 2018-06-01 22:31:16.295421224 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; + +class ZAddress { + static long as_long(Address value) { + if (value == null) { + return 0; + } + return value.asLongValue(); + }; + + static boolean is_null(Address value) { + return value == null; + } + + static boolean is_weak_bad(Address value) { + return (as_long(value) & ZGlobals.ZAddressWeakBadMask()) != 0; + } + + static boolean is_weak_good(Address value) { + return !is_weak_bad(value) && !is_null(value); + } + + static boolean is_weak_good_or_null(Address value) { + return !is_weak_bad(value); + } + + static long offset(Address address) { + return as_long(address) & ZGlobals.ZAddressOffsetMask; + } + + static Address address(long value) { + VM vm = VM.getVM(); + if (vm.getOS().equals("solaris") && vm.getCPU().equals("sparc")) { + value |= ZGlobals.ZAddressSpaceStart; + } + + return ZOop.to_address(value); + } + + static Address good(Address value) { + return address(offset(value) | ZGlobals.ZAddressGoodMask()); + } + + static Address good_or_null(Address value) { + return is_null(value) ? value : good(value); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddressRangeMapForPageTable.java 2018-06-01 22:31:16.673437541 +0200 @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.types.AddressField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +public class ZAddressRangeMapForPageTable extends VMObject { + private static AddressField mapField; + + private static long AddressRangeShift = ZGlobals.ZPageSizeMinShift; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZAddressRangeMapForPageTable"); + + mapField = type.getAddressField("_map"); + } + + public ZAddressRangeMapForPageTable(Address addr) { + super(addr); + } + + private Address map() { + return mapField.getValue(addr); + } + + private long index_for_addr(Address addr) { + long index = ZAddress.offset(addr) >> AddressRangeShift; + + return index; + } + + Address get(Address addr) { + long index = index_for_addr(addr); + + return map().getAddressAt(index * VM.getVM().getBytesPerLong()); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZBarrier.java 2018-06-01 22:31:17.049453771 +0200 @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; + +class ZBarrier { + private static boolean is_weak_good_or_null_fast_path(Address addr) { + return ZAddress.is_weak_good_or_null(addr); + } + + private static Address weak_load_barrier_on_oop_slow_path(Address addr) { + return ZAddress.is_weak_good(addr) ? ZAddress.good(addr) : relocate_or_remap(addr); + } + + private static boolean during_relocate() { + return ZGlobals.ZGlobalPhase() == ZGlobals.ZPhaseRelocate; + } + + private static Address relocate(Address addr) { + ZHeap heap = zheap(); + if (heap.is_relocating(addr)) { + // Forward + return heap.relocate_object(addr); + } + + // Remap + return ZAddress.good(addr); + } + + private static ZHeap zheap() { + ZCollectedHeap zCollectedHeap = (ZCollectedHeap)VM.getVM().getUniverse().heap(); + return zCollectedHeap.heap(); + } + + private static Address remap(Address addr) { + ZHeap heap = zheap(); + if (heap.is_relocating(addr)) { + // Forward + return heap.forward_object(addr); + } + + // Remap + return ZAddress.good(addr); + } + + private static Address relocate_or_remap(Address addr) { + return during_relocate() ? relocate(addr) : remap(addr); + } + + static Address weak_barrier(Address o) { + // Fast path + if (is_weak_good_or_null_fast_path(o)) { + // Return the good address instead of the weak good address + // to ensure that the currently active heap view is used. + return ZAddress.good_or_null(o); + } + + // Slow path + return weak_load_barrier_on_oop_slow_path(o); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java 2018-06-01 22:31:17.414469527 +0200 @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import java.io.PrintStream; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.debugger.OopHandle; +import sun.jvm.hotspot.gc.shared.CollectedHeap; +import sun.jvm.hotspot.gc.shared.CollectedHeapName; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObjectFactory; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +// Mirror class for ZCollectedHeap. + +public class ZCollectedHeap extends CollectedHeap { + + private static long zHeapFieldOffset; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + private static synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZCollectedHeap"); + + zHeapFieldOffset = type.getAddressField("_heap").getOffset(); + } + + public ZHeap heap() { + Address heapAddr = addr.addOffsetTo(zHeapFieldOffset); + return (ZHeap)VMObjectFactory.newObject(ZHeap.class, heapAddr); + } + + @Override + public CollectedHeapName kind() { + return CollectedHeapName.Z; + } + + @Override + public void printOn(PrintStream tty) { + heap().printOn(tty); + } + + public ZCollectedHeap(Address addr) { + super(addr); + } + + public OopHandle oop_load_at(OopHandle handle, long offset) { + assert(!VM.getVM().isCompressedOopsEnabled()); + + Address oopAddress = handle.getAddressAt(offset); + + oopAddress = ZBarrier.weak_barrier(oopAddress); + if (oopAddress == null) { + return null; + } + + return oopAddress.addOffsetToAsOopHandle(0); + } + + public String oopAddressDescription(OopHandle handle) { + Address origOop = ZOop.to_address(handle); + Address loadBarrieredOop = ZBarrier.weak_barrier(origOop); + if (!origOop.equals(loadBarrieredOop)) { + return origOop + " (" + loadBarrieredOop.toString() + ")"; + } else { + return handle.toString(); + } + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTable.java 2018-06-01 22:31:17.788485671 +0200 @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.types.AddressField; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +public class ZForwardingTable extends VMObject { + private static AddressField tableField; + private static CIntegerField sizeField; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZForwardingTable"); + + tableField = type.getAddressField("_table"); + sizeField = type.getCIntegerField("_size"); + } + + public ZForwardingTable(Address addr) { + super(addr); + } + + Address table() { + return tableField.getAddress(addr); + } + + long size() { + return sizeField.getJLong(addr); + } + + ZForwardingTableEntry at(ZForwardingTableCursor cursor) { + return new ZForwardingTableEntry(table().getAddressAt(cursor._value * VM.getVM().getBytesPerLong())); + } + + ZForwardingTableEntry first(long from_index, ZForwardingTableCursor cursor) { + long mask = size() - 1; + long hash = ZHash.uint32_to_uint32(from_index); + cursor._value = hash & mask; + return at(cursor); + } + + ZForwardingTableEntry next(ZForwardingTableCursor cursor) { + long mask = size() - 1; + cursor._value = (cursor._value + 1) & mask; + return at(cursor); + } + + ZForwardingTableEntry find(long from_index, ZForwardingTableCursor cursor) { + // Reading entries in the table races with the atomic cas done for + // insertion into the table. This is safe because each entry is at + // most updated once (from -1 to something else). + ZForwardingTableEntry entry = first(from_index, cursor); + while (!entry.is_empty()) { + if (entry.from_index() == from_index) { + // Match found, return matching entry + return entry; + } + + entry = next(cursor); + } + + // Match not found, return empty entry + return entry; + } + + ZForwardingTableEntry find(long from_index) { + ZForwardingTableCursor dummy = new ZForwardingTableCursor(); + return find(from_index, dummy); + } + + void dump() { + long s = size(); + long count = 0; + System.out.println("Dumping ZForwardingTable[" + s + "]:"); + ZForwardingTableCursor cursor = new ZForwardingTableCursor(); + for (long i = 0; i < s; i++) { + cursor._value = i; + ZForwardingTableEntry entry = at(cursor); + if (!entry.is_empty()) { + long hash = ZHash.uint32_to_uint32(entry.from_index()); + System.out.println(i + " " + count + " " + entry + " hash: " + hash + " masked_hash: " + (hash & (s - 1))); + count++; + } + } + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableCursor.java 2018-06-01 22:31:18.169502117 +0200 @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +class ZForwardingTableCursor { + long _value; +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableEntry.java 2018-06-01 22:31:18.562519082 +0200 @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; + +class ZForwardingTableEntry { + private Address entry; + + ZForwardingTableEntry(Address addr) { + entry = addr; + } + + private static long empty() { + return ~0L; + } + + boolean is_empty() { + return entry.asLongValue() == empty(); + } + + Address to_offset() { + return entry.andWithMask((1L << 42) - 1); + } + + long from_index() { + return entry.asLongValue() >>> 42; + } + + public String toString() { + return entry + " - from_index: " + from_index() + " to_offset: " + to_offset(); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java 2018-06-01 22:31:18.956536089 +0200 @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.types.Field; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +public class ZGlobals { + private static Field instanceField; + + // Global phase state + public static int ZPhaseRelocate; + + public static byte ZPageTypeSmall; + public static byte ZPageTypeMedium; + public static byte ZPageTypeLarge; + + // Page size shifts + public static long ZPageSizeSmallShift; + public static long ZPageSizeMediumShift; + public static long ZPageSizeMinShift; + + // Object alignment shifts + public static int ZObjectAlignmentMediumShift; + public static int ZObjectAlignmentLargeShift; + + // Pointer part of address + public static long ZAddressOffsetShift; + + // Pointer part of address + public static long ZAddressOffsetBits; + public static long ZAddressOffsetMask; + + // Address space start/end/size + public static long ZAddressSpaceStart; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZGlobalsForVMStructs"); + + instanceField = type.getField("_instance_p"); + + ZPhaseRelocate = db.lookupIntConstant("ZPhaseRelocate").intValue(); + + ZPageTypeSmall = db.lookupIntConstant("ZPageTypeSmall").byteValue(); + ZPageTypeMedium = db.lookupIntConstant("ZPageTypeMedium").byteValue(); + ZPageTypeLarge = db.lookupIntConstant("ZPageTypeLarge").byteValue(); + + ZPageSizeSmallShift = db.lookupLongConstant("ZPageSizeSmallShift").longValue(); + ZPageSizeMediumShift = db.lookupLongConstant("ZPageSizeMediumShift").longValue(); + ZPageSizeMinShift = db.lookupLongConstant("ZPageSizeMinShift").longValue(); + + ZObjectAlignmentMediumShift = db.lookupIntConstant("ZObjectAlignmentMediumShift").intValue(); + ZObjectAlignmentLargeShift = db.lookupIntConstant("ZObjectAlignmentLargeShift").intValue();; + + ZAddressOffsetShift = db.lookupLongConstant("ZAddressOffsetShift").longValue(); + + ZAddressOffsetBits = db.lookupLongConstant("ZAddressOffsetBits").longValue(); + ZAddressOffsetMask = db.lookupLongConstant("ZAddressOffsetMask").longValue(); + + ZAddressSpaceStart = db.lookupLongConstant("ZAddressSpaceStart").longValue(); + } + + private static ZGlobalsForVMStructs instance() { + return new ZGlobalsForVMStructs(instanceField.getAddress()); + } + + public static int ZGlobalPhase() { + return instance().ZGlobalPhase(); + } + + public static long ZAddressGoodMask() { + return instance().ZAddressGoodMask(); + } + + public static long ZAddressBadMask() { + return instance().ZAddressBadMask(); + } + + public static long ZAddressWeakBadMask() { + return instance().ZAddressWeakBadMask(); + } + + public static int ZObjectAlignmentSmallShift() { + return instance().ZObjectAlignmentSmallShift(); + } + + public static int ZObjectAlignmentSmall() { + return instance().ZObjectAlignmentSmall(); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java 2018-06-01 22:31:19.332552319 +0200 @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.types.AddressField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +class ZGlobalsForVMStructs extends VMObject { + private static AddressField ZGlobalPhaseField; + private static AddressField ZAddressGoodMaskField; + private static AddressField ZAddressBadMaskField; + private static AddressField ZAddressWeakBadMaskField; + private static AddressField ZObjectAlignmentSmallShiftField; + private static AddressField ZObjectAlignmentSmallField; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZGlobalsForVMStructs"); + + ZGlobalPhaseField = type.getAddressField("_ZGlobalPhase"); + ZAddressGoodMaskField = type.getAddressField("_ZAddressGoodMask"); + ZAddressBadMaskField = type.getAddressField("_ZAddressBadMask"); + ZAddressWeakBadMaskField = type.getAddressField("_ZAddressWeakBadMask"); + ZObjectAlignmentSmallShiftField = type.getAddressField("_ZObjectAlignmentSmallShift"); + ZObjectAlignmentSmallField = type.getAddressField("_ZObjectAlignmentSmall"); + } + + ZGlobalsForVMStructs(Address addr) { + super(addr); + } + + int ZGlobalPhase() { + return ZGlobalPhaseField.getValue(addr).getJIntAt(0); + } + + long ZAddressGoodMask() { + return ZAddressGoodMaskField.getValue(addr).getJLongAt(0); + } + + long ZAddressBadMask() { + return ZAddressBadMaskField.getValue(addr).getJLongAt(0); + } + + long ZAddressWeakBadMask() { + return ZAddressWeakBadMaskField.getValue(addr).getJLongAt(0); + } + + int ZObjectAlignmentSmallShift() { + return ZObjectAlignmentSmallShiftField.getValue(addr).getJIntAt(0); + } + + int ZObjectAlignmentSmall() { + return ZObjectAlignmentSmallField.getValue(addr).getJIntAt(0); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHash.java 2018-06-01 22:31:19.715568852 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +class ZHash { + private static long uint32(long value) { + return value & 0xFFFFFFFFL; + } + + static long uint32_to_uint32(long key) { + key = uint32(~key + (key << 15)); + key = uint32(key ^ (key >> 12)); + key = uint32(key + (key << 2)); + key = uint32(key ^ (key >> 4)); + key = uint32(key * 2057); + key = uint32(key ^ (key >> 16)); + return key; + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHeap.java 2018-06-01 22:31:20.108585816 +0200 @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import java.io.PrintStream; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.runtime.VMObjectFactory; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +// Mirror class for ZHeap + +public class ZHeap extends VMObject { + + private static long pageAllocatorFieldOffset; + private static long pageTableFieldOffset; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + private static synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZHeap"); + + pageAllocatorFieldOffset = type.getAddressField("_page_allocator").getOffset(); + pageTableFieldOffset = type.getAddressField("_pagetable").getOffset(); + } + + public ZHeap(Address addr) { + super(addr); + } + + private ZPageAllocator pageAllocator() { + Address pageAllocatorAddr = addr.addOffsetTo(pageAllocatorFieldOffset); + return (ZPageAllocator)VMObjectFactory.newObject(ZPageAllocator.class, pageAllocatorAddr); + } + + ZPageTable pageTable() { + return (ZPageTable)VMObjectFactory.newObject(ZPageTable.class, addr.addOffsetTo(pageTableFieldOffset)); + } + + public long maxCapacity() { + return pageAllocator().maxCapacity(); + } + + public long capacity() { + return pageAllocator().capacity(); + } + + public long used() { + return pageAllocator().used(); + } + + boolean is_relocating(Address o) { + return pageTable().is_relocating(o); + } + + Address forward_object(Address addr) { + ZPage page = pageTable().get(addr); + return page.forward_object(addr); + } + + Address relocate_object(Address addr) { + ZPage page = pageTable().get(addr); + return page.relocate_object(addr); + } + + public void printOn(PrintStream tty) { + tty.print(" ZHeap "); + tty.print("used " + (used() / 1024 / 1024) + "M, "); + tty.print("capacity " + (capacity() / 1024 / 1024) + "M, "); + tty.println("max capacity " + (maxCapacity() / 1024 / 1024) + "M"); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZOop.java 2018-06-01 22:31:20.485602090 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.debugger.OopHandle; +import sun.jvm.hotspot.runtime.VM; + +class ZOop { + private static final long MSB = ~0L ^ (~0L >>> 1); + + private static Address msbAddress() { + return VM.getVM().getUniverse().heap().start().orWithMask(MSB).andWithMask(MSB); + } + + static Address to_address(long value) { + // If the value of an Address becomes 0, null is returned instead of an Address. + // Start with a one-bit address and as a last step, remove that bit. + Address oneAddress = msbAddress(); + return oneAddress.orWithMask(value).xorWithMask(ZAddress.as_long(oneAddress)); + } + + static Address to_address(OopHandle oop) { + return to_address(ZAddress.as_long(oop)); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPage.java 2018-06-01 22:31:20.860618277 +0200 @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.runtime.VMObjectFactory; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +public class ZPage extends VMObject { + private static CIntegerField typeField; + private static long virtualFieldOffset; + private static long forwardingFieldOffset; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZPage"); + + typeField = type.getCIntegerField("_type"); + virtualFieldOffset = type.getField("_virtual").getOffset(); + forwardingFieldOffset = type.getField("_forwarding").getOffset(); + } + + public ZPage(Address addr) { + super(addr); + } + + private byte type() { + return typeField.getJByte(addr); + } + + private ZVirtualMemory virtual() { + return (ZVirtualMemory)VMObjectFactory.newObject(ZVirtualMemory.class, addr.addOffsetTo(virtualFieldOffset)); + } + + private ZForwardingTable forwarding() { + return (ZForwardingTable)VMObjectFactory.newObject(ZForwardingTable.class, addr.addOffsetTo(forwardingFieldOffset)); + } + + private long start() { + return virtual().start(); + } + + Address forward_object(Address from) { + // Lookup address in forwarding table + long from_offset = ZAddress.offset(from); + long from_index = (from_offset - start()) >> object_alignment_shift(); + ZForwardingTableEntry entry = forwarding().find(from_index); + assert(!entry.is_empty()); + assert(entry.from_index() == from_index); + + return ZAddress.good(entry.to_offset()); + } + + Address relocate_object(Address from) { + // Lookup address in forwarding table + long from_offset = ZAddress.offset(from); + long from_index = (from_offset - start()) >> object_alignment_shift(); + ZForwardingTableEntry entry = forwarding().find(from_index); + if (!entry.is_empty() && entry.from_index() == from_index) { + return ZAddress.good(entry.to_offset()); + } + + // There's no relocate operation in the SA. + // Mimic object pinning and return the good view of the from object. + return ZAddress.good(from); + } + + + long object_alignment_shift() { + if (type() == ZGlobals.ZPageTypeSmall) { + return ZGlobals.ZObjectAlignmentSmallShift(); + } else if (type() == ZGlobals.ZPageTypeMedium) { + return ZGlobals.ZObjectAlignmentMediumShift; + } else { + assert(type() == ZGlobals.ZPageTypeLarge); + return ZGlobals.ZObjectAlignmentLargeShift; + } + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageAllocator.java 2018-06-01 22:31:21.241634723 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.runtime.VMObjectFactory; +import sun.jvm.hotspot.types.AddressField; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +// Mirror class for ZPageAllocator + +public class ZPageAllocator extends VMObject { + + private static AddressField physicalField; + private static CIntegerField usedField; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZPageAllocator"); + + physicalField = type.getAddressField("_physical"); + usedField = type.getCIntegerField("_used"); + } + + private ZPhysicalMemoryManager physical() { + Address physicalAddr = physicalField.getValue(addr); + return (ZPhysicalMemoryManager)VMObjectFactory.newObject(ZPhysicalMemoryManager.class, physicalAddr); + } + + public long maxCapacity() { + return physical().maxCapacity(); + } + + public long capacity() { + return physical().capacity(); + } + + public long used() { + return usedField.getValue(addr); + } + + public ZPageAllocator(Address addr) { + super(addr); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTable.java 2018-06-01 22:31:21.610650652 +0200 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.runtime.VMObjectFactory; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +public class ZPageTable extends VMObject { + private static long mapFieldOffset; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZPageTable"); + + mapFieldOffset = type.getAddressField("_map").getOffset(); + } + + public ZPageTable(Address addr) { + super(addr); + } + + private ZAddressRangeMapForPageTable map() { + return (ZAddressRangeMapForPageTable)VMObjectFactory.newObject(ZAddressRangeMapForPageTable.class, addr.addOffsetTo(mapFieldOffset)); + } + + private ZPageTableEntry getEntry(Address o) { + return new ZPageTableEntry(map().get(o)); + } + + ZPage get(Address o) { + return getEntry(o).page(); + } + + boolean is_relocating(Address o) { + return getEntry(o).relocating(); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTableEntry.java 2018-06-01 22:31:21.977666493 +0200 @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VMObjectFactory; + +class ZPageTableEntry { + Address entry; + + ZPageTableEntry(Address address) { + entry = address; + } + + ZPage page() { + return (ZPage)VMObjectFactory.newObject(ZPage.class, entry.andWithMask(~1L)); + } + + boolean relocating() { + return (entry.asLongValue() & 1) == 1; + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPhysicalMemoryManager.java 2018-06-01 22:31:22.358682940 +0200 @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +// Mirror class for ZPhysicalMemoryManager + +public class ZPhysicalMemoryManager extends VMObject { + + private static CIntegerField capacityField; + + private static CIntegerField maxCapacityField; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + private static synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZPhysicalMemoryManager"); + + capacityField = type.getCIntegerField("_capacity"); + maxCapacityField = type.getCIntegerField("_max_capacity"); + } + + public long capacity() { + return capacityField.getValue(addr); + } + + public long maxCapacity() { + return maxCapacityField.getValue(addr); + } + + public ZPhysicalMemoryManager(Address addr) { + super(addr); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZVirtualMemory.java 2018-06-01 22:31:22.745699645 +0200 @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.z; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +public class ZVirtualMemory extends VMObject { + private static CIntegerField startField; + private static CIntegerField endField; + + static { + VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase())); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ZVirtualMemory"); + + startField = type.getCIntegerField("_start"); + endField = type.getCIntegerField("_end"); + } + + public ZVirtualMemory(Address addr) { + super(addr); + } + + long start() { + return startField.getJLong(addr); + } + + long end() { + return endField.getJLong(addr); + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zAddress.cpp 2018-06-01 22:31:23.129716221 +0200 @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zGlobals.hpp" +#include "unittest.hpp" + +class ZAddressTest : public ::testing::Test { +protected: + static void is_good_bit(uintptr_t bit_mask) { + // Setup + uintptr_t mask_before = ZAddressGoodMask; + + ZAddressMasks::set_good_mask(bit_mask); + + // Test that a pointer with only the given bit is considered good. + EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked0), (bit_mask == ZAddressMetadataMarked0)); + EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked1), (bit_mask == ZAddressMetadataMarked1)); + EXPECT_EQ(ZAddress::is_good(ZAddressMetadataRemapped), (bit_mask == ZAddressMetadataRemapped)); + + // Test that a pointer with the given bit and some extra bits is considered good. + EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked0 | 0x8),(bit_mask == ZAddressMetadataMarked0)); + EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked1 | 0x8), (bit_mask == ZAddressMetadataMarked1)); + EXPECT_EQ(ZAddress::is_good(ZAddressMetadataRemapped | 0x8), (bit_mask == ZAddressMetadataRemapped)); + + // Test that null is not considered good. + EXPECT_FALSE(ZAddress::is_good(0)); + + // Teardown + ZAddressMasks::set_good_mask(mask_before); + } + + static void is_good_or_null_bit(uintptr_t bit_mask) { + // Setup + uintptr_t mask_before = ZAddressGoodMask; + + ZAddressMasks::set_good_mask(bit_mask); + + // Test that a pointer with only the given bit is considered good. + EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked0), (bit_mask == ZAddressMetadataMarked0)); + EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked1), (bit_mask == ZAddressMetadataMarked1)); + EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataRemapped), (bit_mask == ZAddressMetadataRemapped)); + + // Test that a pointer with the given bit and some extra bits is considered good. + EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked0 | 0x8), (bit_mask == ZAddressMetadataMarked0)); + EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked1 | 0x8), (bit_mask == ZAddressMetadataMarked1)); + EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataRemapped | 0x8), (bit_mask == ZAddressMetadataRemapped)); + + // Test that null is considered good_or_null. + EXPECT_TRUE(ZAddress::is_good_or_null(0)); + + // Teardown + ZAddressMasks::set_good_mask(mask_before); + } + + static void finalizable() { + // Setup + ZAddressMasks::initialize(); + ZAddressMasks::flip_to_marked(); + + // Test that a normal good pointer is good and weak good, but not finalizable + const uintptr_t addr1 = ZAddress::good(1); + EXPECT_FALSE(ZAddress::is_finalizable(addr1)); + EXPECT_TRUE(ZAddress::is_marked(addr1)); + EXPECT_FALSE(ZAddress::is_remapped(addr1)); + EXPECT_TRUE(ZAddress::is_weak_good(addr1)); + EXPECT_TRUE(ZAddress::is_weak_good_or_null(addr1)); + EXPECT_TRUE(ZAddress::is_good(addr1)); + EXPECT_TRUE(ZAddress::is_good_or_null(addr1)); + + // Test that a finalizable good pointer is finalizable and weak good, but not good + const uintptr_t addr2 = ZAddress::finalizable_good(1); + EXPECT_TRUE(ZAddress::is_finalizable(addr2)); + EXPECT_TRUE(ZAddress::is_marked(addr2)); + EXPECT_FALSE(ZAddress::is_remapped(addr2)); + EXPECT_TRUE(ZAddress::is_weak_good(addr2)); + EXPECT_TRUE(ZAddress::is_weak_good_or_null(addr2)); + EXPECT_FALSE(ZAddress::is_good(addr2)); + EXPECT_FALSE(ZAddress::is_good_or_null(addr2)); + + // Flip to remapped and test that it's no longer weak good + ZAddressMasks::flip_to_remapped(); + EXPECT_TRUE(ZAddress::is_finalizable(addr2)); + EXPECT_TRUE(ZAddress::is_marked(addr2)); + EXPECT_FALSE(ZAddress::is_remapped(addr2)); + EXPECT_FALSE(ZAddress::is_weak_good(addr2)); + EXPECT_FALSE(ZAddress::is_weak_good_or_null(addr2)); + EXPECT_FALSE(ZAddress::is_good(addr2)); + EXPECT_FALSE(ZAddress::is_good_or_null(addr2)); + } +}; + +TEST_F(ZAddressTest, is_good) { + is_good_bit(ZAddressMetadataMarked0); + is_good_bit(ZAddressMetadataMarked1); + is_good_bit(ZAddressMetadataRemapped); +} + +TEST_F(ZAddressTest, is_good_or_null) { + is_good_or_null_bit(ZAddressMetadataMarked0); + is_good_or_null_bit(ZAddressMetadataMarked1); + is_good_or_null_bit(ZAddressMetadataRemapped); +} + +TEST_F(ZAddressTest, is_weak_good_or_null) { +#define check_is_weak_good_or_null(value) \ + EXPECT_EQ(ZAddress::is_weak_good_or_null(value), \ + (ZAddress::is_good_or_null(value) || ZAddress::is_remapped(value))) \ + << "is_good_or_null: " << ZAddress::is_good_or_null(value) \ + << " is_remaped: " << ZAddress::is_remapped(value) \ + << " is_good_or_null_or_remapped: " << ZAddress::is_weak_good_or_null(value) + + check_is_weak_good_or_null((uintptr_t)NULL); + check_is_weak_good_or_null(ZAddressMetadataMarked0); + check_is_weak_good_or_null(ZAddressMetadataMarked1); + check_is_weak_good_or_null(ZAddressMetadataRemapped); + check_is_weak_good_or_null((uintptr_t)0x123); +} + +TEST_F(ZAddressTest, finalizable) { + finalizable(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zArray.cpp 2018-06-01 22:31:23.505732451 +0200 @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zArray.inline.hpp" +#include "unittest.hpp" + +TEST(ZArrayTest, test_add) { + ZArray a; + + // Add elements + for (int i = 0; i < 10; i++) { + a.add(i); + } + + // Check size + ASSERT_EQ(a.size(), 10u); + + // Check elements + for (int i = 0; i < 10; i++) { + EXPECT_EQ(a.at(i), i); + } +} + +TEST(ZArrayTest, test_clear) { + ZArray a; + + // Add elements + for (int i = 0; i < 10; i++) { + a.add(i); + } + + // Check size + ASSERT_EQ(a.size(), 10u); + ASSERT_EQ(a.is_empty(), false); + + // Clear elements + a.clear(); + + // Check size + ASSERT_EQ(a.size(), 0u); + ASSERT_EQ(a.is_empty(), true); + + // Add element + a.add(11); + + // Check size + ASSERT_EQ(a.size(), 1u); + ASSERT_EQ(a.is_empty(), false); + + // Clear elements + a.clear(); + + // Check size + ASSERT_EQ(a.size(), 0u); + ASSERT_EQ(a.is_empty(), true); +} + +TEST(ZArrayTest, test_iterator) { + ZArray a; + + // Add elements + for (int i = 0; i < 10; i++) { + a.add(i); + } + + // Iterate + int count = 0; + ZArrayIterator iter(&a); + for (int value; iter.next(&value);) { + ASSERT_EQ(a.at(count), count); + count++; + } + + // Check count + ASSERT_EQ(count, 10); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zBitField.cpp 2018-06-01 22:31:23.866748034 +0200 @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zBitField.hpp" +#include "unittest.hpp" + +TEST(ZBitFieldTest, test) { + typedef ZBitField field_bool; + typedef ZBitField field_uint8; + typedef ZBitField field_uint16; + typedef ZBitField field_uint32; + typedef ZBitField field_uint64; + typedef ZBitField field_pointer; + + uint64_t entry; + + { + const bool value = false; + entry = field_bool::encode(value); + EXPECT_EQ(field_bool::decode(entry), value) << "Should be equal"; + } + + { + const bool value = true; + entry = field_bool::encode(value); + EXPECT_EQ(field_bool::decode(entry), value) << "Should be equal"; + } + + { + const uint8_t value = ~(uint8_t)0; + entry = field_uint8::encode(value); + EXPECT_EQ(field_uint8::decode(entry), value) << "Should be equal"; + } + + { + const uint16_t value = ~(uint16_t)0; + entry = field_uint16::encode(value); + EXPECT_EQ(field_uint16::decode(entry), value) << "Should be equal"; + } + + { + const uint32_t value = ~(uint32_t)0; + entry = field_uint32::encode(value); + EXPECT_EQ(field_uint32::decode(entry), value) << "Should be equal"; + } + + { + const uint64_t value = ~(uint64_t)0 >> 1; + entry = field_uint64::encode(value); + EXPECT_EQ(field_uint64::decode(entry), value) << "Should be equal"; + } + + { + void* const value = (void*)(~(uintptr_t)0 << 3); + entry = field_pointer::encode(value); + EXPECT_EQ(field_pointer::decode(entry), value) << "Should be equal"; + } +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zBitMap.cpp 2018-06-01 22:31:24.242764264 +0200 @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zBitMap.inline.hpp" +#include "unittest.hpp" + +class ZBitMapTest : public ::testing::Test { +protected: + static void test_set_pair_unset(size_t size, bool finalizable) { + ZBitMap bitmap(size); + + for (BitMap::idx_t i = 0; i < size - 1; i++) { + if ((i + 1) % BitsPerWord == 0) { + // Can't set pairs of bits in different words. + continue; + } + + // ZBitMaps are not cleared when constructed. + bitmap.clear(); + + bool inc_live = false; + + bool ret = bitmap.par_set_bit_pair(i, finalizable, inc_live); + EXPECT_TRUE(ret) << "Failed to set bit"; + EXPECT_TRUE(inc_live) << "Should have set inc_live"; + + // First bit should always be set + EXPECT_TRUE(bitmap.at(i)) << "Should be set"; + + // Second bit should only be set when marking strong + EXPECT_NE(bitmap.at(i + 1), finalizable); + } + } + + static void test_set_pair_set(size_t size, bool finalizable) { + ZBitMap bitmap(size); + + for (BitMap::idx_t i = 0; i < size - 1; i++) { + if ((i + 1) % BitsPerWord == 0) { + // Can't set pairs of bits in different words. + continue; + } + + // Fill the bitmap with ones. + bitmap.set_range(0, size); + + bool inc_live = false; + + bool ret = bitmap.par_set_bit_pair(i, finalizable, inc_live); + EXPECT_FALSE(ret) << "Should not succeed setting bit"; + EXPECT_FALSE(inc_live) << "Should not have set inc_live"; + + // Both bits were pre-set. + EXPECT_TRUE(bitmap.at(i)) << "Should be set"; + EXPECT_TRUE(bitmap.at(i + 1)) << "Should be set"; + } + } + + static void test_set_pair_set(bool finalizable) { + test_set_pair_set(2, finalizable); + test_set_pair_set(62, finalizable); + test_set_pair_set(64, finalizable); + test_set_pair_set(66, finalizable); + test_set_pair_set(126, finalizable); + test_set_pair_set(128, finalizable); + } + + static void test_set_pair_unset(bool finalizable) { + test_set_pair_unset(2, finalizable); + test_set_pair_unset(62, finalizable); + test_set_pair_unset(64, finalizable); + test_set_pair_unset(66, finalizable); + test_set_pair_unset(126, finalizable); + test_set_pair_unset(128, finalizable); + } + +}; + +TEST_F(ZBitMapTest, test_set_pair_set) { + test_set_pair_set(false); + test_set_pair_set(true); +} + +TEST_F(ZBitMapTest, test_set_pair_unset) { + test_set_pair_unset(false); + test_set_pair_unset(true); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zForwardingTable.cpp 2018-06-01 22:31:24.620780581 +0200 @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zForwardingTable.inline.hpp" +#include "unittest.hpp" + +using namespace testing; + +#define CAPTURE_DELIM "\n" +#define CAPTURE1(expression) #expression << " evaluates to " << expression +#define CAPTURE2(e0, e1) CAPTURE1(e0) << CAPTURE_DELIM << CAPTURE1(e1) + +#define CAPTURE(expression) CAPTURE1(expression) + +class ZForwardingTableTest : public Test { +public: + // Helper functions + + static bool is_power_of_2(size_t value) { + return ::is_power_of_2((intptr_t)value); + } + + class SequenceToFromIndex : AllStatic { + public: + static uintptr_t even(uint32_t sequence_number) { + return sequence_number * 2; + } + static uintptr_t odd(uint32_t sequence_number) { + return even(sequence_number) + 1; + } + static uintptr_t one_to_one(uint32_t sequence_number) { + return sequence_number; + } + }; + + // Test functions + + static void setup(ZForwardingTable& table) { + EXPECT_PRED1(is_power_of_2, table._size) << CAPTURE(table._size); + } + + static void find_empty(ZForwardingTable& table) { + size_t size = table._size; + size_t entries_to_check = size * 2; + + for (uint32_t i = 0; i < entries_to_check; i++) { + uintptr_t from_index = SequenceToFromIndex::one_to_one(i); + + EXPECT_TRUE(table.find(from_index).is_empty()) << CAPTURE2(from_index, size); + } + + EXPECT_TRUE(table.find(uintptr_t(-1)).is_empty()) << CAPTURE(size); + } + + static void find_full(ZForwardingTable& table) { + size_t size = table._size; + size_t entries_to_populate = size; + + // Populate + for (uint32_t i = 0; i < entries_to_populate; i++) { + uintptr_t from_index = SequenceToFromIndex::one_to_one(i); + + ZForwardingTableCursor cursor; + ZForwardingTableEntry entry = table.find(from_index, &cursor); + ASSERT_TRUE(entry.is_empty()) << CAPTURE2(from_index, size); + + table.insert(from_index, from_index, &cursor); + } + + // Verify + for (uint32_t i = 0; i < entries_to_populate; i++) { + uintptr_t from_index = SequenceToFromIndex::one_to_one(i); + + ZForwardingTableEntry entry = table.find(from_index); + ASSERT_FALSE(entry.is_empty()) << CAPTURE2(from_index, size); + + ASSERT_EQ(entry.from_index(), from_index) << CAPTURE(size); + ASSERT_EQ(entry.to_offset(), from_index) << CAPTURE(size); + } + } + + static void find_every_other(ZForwardingTable& table) { + size_t size = table._size; + size_t entries_to_populate = size / 2; + + // Populate even from indices + for (uint32_t i = 0; i < entries_to_populate; i++) { + uintptr_t from_index = SequenceToFromIndex::even(i); + + ZForwardingTableCursor cursor; + ZForwardingTableEntry entry = table.find(from_index, &cursor); + ASSERT_TRUE(entry.is_empty()) << CAPTURE2(from_index, size); + + table.insert(from_index, from_index, &cursor); + } + + // Verify populated even indices + for (uint32_t i = 0; i < entries_to_populate; i++) { + uintptr_t from_index = SequenceToFromIndex::even(i); + + ZForwardingTableCursor cursor; + ZForwardingTableEntry entry = table.find(from_index, &cursor); + ASSERT_FALSE(entry.is_empty()) << CAPTURE2(from_index, size); + + ASSERT_EQ(entry.from_index(), from_index) << CAPTURE(size); + ASSERT_EQ(entry.to_offset(), from_index) << CAPTURE(size); + } + + // Verify empty odd indices + // + // This check could be done on a larger range of sequence numbers, + // but currently entries_to_populate is used. + for (uint32_t i = 0; i < entries_to_populate; i++) { + uintptr_t from_index = SequenceToFromIndex::odd(i); + + ZForwardingTableEntry entry = table.find(from_index); + + ASSERT_TRUE(entry.is_empty()) << CAPTURE2(from_index, size); + } + } + + static void test(void (*function)(ZForwardingTable&), uint32_t size) { + // Setup + ZForwardingTable table; + table.setup(size); + ASSERT_FALSE(table.is_null()); + + // Actual test function + (*function)(table); + + // Teardown + table.reset(); + ASSERT_TRUE(table.is_null()); + } + + // Run the given function with a few different input values. + static void test(void (*function)(ZForwardingTable&)) { + test(function, 1); + test(function, 2); + test(function, 3); + test(function, 4); + test(function, 7); + test(function, 8); + test(function, 1023); + test(function, 1024); + test(function, 1025); + } +}; + +TEST_F(ZForwardingTableTest, setup) { + test(&ZForwardingTableTest::setup); +} + +TEST_F(ZForwardingTableTest, find_empty) { + test(&ZForwardingTableTest::find_empty); +} + +TEST_F(ZForwardingTableTest, find_full) { + test(&ZForwardingTableTest::find_full); +} + +TEST_F(ZForwardingTableTest, find_every_other) { + test(&ZForwardingTableTest::find_every_other); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zList.cpp 2018-06-01 22:31:24.990796553 +0200 @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zList.inline.hpp" +#include "unittest.hpp" + +#ifndef PRODUCT + +class ZTestEntry { + friend class ZList; + +private: + const int _id; + ZListNode _node; + +public: + ZTestEntry(int id) : + _id(id), + _node() {} + + int id() const { + return _id; + } +}; + +class ZListTest : public ::testing::Test { +protected: + static void assert_sorted(ZList* list) { + // Iterate forward + { + int count = list->first()->id(); + ZListIterator iter(list); + for (ZTestEntry* entry; iter.next(&entry);) { + ASSERT_EQ(entry->id(), count); + count++; + } + } + + // Iterate backward + { + int count = list->last()->id(); + ZListReverseIterator iter(list); + for (ZTestEntry* entry; iter.next(&entry);) { + EXPECT_EQ(entry->id(), count); + count--; + } + } + } +}; + +TEST_F(ZListTest, test_insert) { + ZList list; + ZTestEntry e0(0); + ZTestEntry e1(1); + ZTestEntry e2(2); + ZTestEntry e3(3); + ZTestEntry e4(4); + ZTestEntry e5(5); + + list.insert_first(&e2); + list.insert_before(&e2, &e1); + list.insert_after(&e2, &e3); + list.insert_last(&e4); + list.insert_first(&e0); + list.insert_last(&e5); + + EXPECT_EQ(list.size(), 6u); + assert_sorted(&list); +} + +TEST_F(ZListTest, test_remove) { + // Remove first + { + ZList list; + ZTestEntry e0(0); + ZTestEntry e1(1); + ZTestEntry e2(2); + ZTestEntry e3(3); + ZTestEntry e4(4); + ZTestEntry e5(5); + + list.insert_last(&e0); + list.insert_last(&e1); + list.insert_last(&e2); + list.insert_last(&e3); + list.insert_last(&e4); + list.insert_last(&e5); + + EXPECT_EQ(list.size(), 6u); + + for (int i = 0; i < 6; i++) { + ZTestEntry* e = list.remove_first(); + EXPECT_EQ(e->id(), i); + } + + EXPECT_EQ(list.size(), 0u); + } + + // Remove last + { + ZList list; + ZTestEntry e0(0); + ZTestEntry e1(1); + ZTestEntry e2(2); + ZTestEntry e3(3); + ZTestEntry e4(4); + ZTestEntry e5(5); + + list.insert_last(&e0); + list.insert_last(&e1); + list.insert_last(&e2); + list.insert_last(&e3); + list.insert_last(&e4); + list.insert_last(&e5); + + EXPECT_EQ(list.size(), 6u); + + for (int i = 5; i >= 0; i--) { + ZTestEntry* e = list.remove_last(); + EXPECT_EQ(e->id(), i); + } + + EXPECT_EQ(list.size(), 0u); + } +} + +TEST_F(ZListTest, test_transfer) { + // Transfer empty to empty + { + ZList list0; + ZList list1; + + EXPECT_TRUE(list0.is_empty()); + EXPECT_TRUE(list1.is_empty()); + + list0.transfer(&list1); + + EXPECT_TRUE(list0.is_empty()); + EXPECT_TRUE(list1.is_empty()); + } + + // Transfer non-empty to empty + { + ZList list0; + ZList list1; + ZTestEntry e0(0); + ZTestEntry e1(1); + ZTestEntry e2(2); + ZTestEntry e3(3); + ZTestEntry e4(4); + ZTestEntry e5(5); + + list1.insert_last(&e0); + list1.insert_last(&e1); + list1.insert_last(&e2); + list1.insert_last(&e3); + list1.insert_last(&e4); + list1.insert_last(&e5); + + EXPECT_EQ(list0.size(), 0u); + EXPECT_EQ(list1.size(), 6u); + + list0.transfer(&list1); + + EXPECT_EQ(list0.size(), 6u); + EXPECT_EQ(list1.size(), 0u); + + assert_sorted(&list0); + } + + // Transfer non-empty to non-empty + { + ZList list0; + ZList list1; + ZTestEntry e0(0); + ZTestEntry e1(1); + ZTestEntry e2(2); + ZTestEntry e3(3); + ZTestEntry e4(4); + ZTestEntry e5(5); + + list0.insert_last(&e0); + list0.insert_last(&e1); + list0.insert_last(&e2); + + list1.insert_last(&e3); + list1.insert_last(&e4); + list1.insert_last(&e5); + + EXPECT_EQ(list0.size(), 3u); + EXPECT_EQ(list1.size(), 3u); + + list0.transfer(&list1); + + EXPECT_EQ(list0.size(), 6u); + EXPECT_EQ(list1.size(), 0u); + + assert_sorted(&list0); + } +} + +#endif // PRODUCT --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zLiveMap.cpp 2018-06-01 22:31:25.363812653 +0200 @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zLiveMap.inline.hpp" +#include "unittest.hpp" + +class ZLiveMapTest : public ::testing::Test { +protected: + static void strongly_live_for_large_zpage() { + // Large ZPages only have room for one object. + ZLiveMap livemap(1); + + bool inc_live; + uintptr_t object = 0u; + + // Mark the object strong. + livemap.set_atomic(object, false /* finalizable */, inc_live); + + // Check that both bits are in the same segment. + ASSERT_EQ(livemap.index_to_segment(0), livemap.index_to_segment(1)); + + // Check that the object was marked. + ASSERT_TRUE(livemap.get(0)); + + // Check that the object was strongly marked. + ASSERT_TRUE(livemap.get(1)); + + ASSERT_TRUE(inc_live); + } +}; + +TEST_F(ZLiveMapTest, strongly_live_for_large_zpage) { + strongly_live_for_large_zpage(); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp 2018-06-01 22:31:25.736828754 +0200 @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zPhysicalMemory.inline.hpp" +#include "utilities/debug.hpp" +#include "unittest.hpp" + +#if defined(AMD64) + +TEST(ZPhysicalMemorySegmentTest, split) { + const size_t SegmentSize = 2 * M; + + ZPhysicalMemorySegment seg(0, 10 * SegmentSize); + + ZPhysicalMemorySegment seg_split0 = seg.split(0 * SegmentSize); + EXPECT_EQ(seg_split0.size(), 0 * SegmentSize); + EXPECT_EQ( seg.size(), 10 * SegmentSize); + + ZPhysicalMemorySegment seg_split1 = seg.split(5 * SegmentSize); + EXPECT_EQ(seg_split1.size(), 5 * SegmentSize); + EXPECT_EQ( seg.size(), 5 * SegmentSize); + + ZPhysicalMemorySegment seg_split2 = seg.split(5 * SegmentSize); + EXPECT_EQ(seg_split2.size(), 5 * SegmentSize); + EXPECT_EQ( seg.size(), 0 * SegmentSize); + + ZPhysicalMemorySegment seg_split3 = seg.split(0 * SegmentSize); + EXPECT_EQ(seg_split3.size(), 0 * SegmentSize); + EXPECT_EQ( seg.size(), 0 * SegmentSize); +} + +TEST(ZPhysicalMemoryTest, split) { + const size_t SegmentSize = 2 * M; + + ZPhysicalMemoryManager pmem_manager(10 * SegmentSize, SegmentSize); + + ZPhysicalMemory pmem = pmem_manager.alloc(8 * SegmentSize); + EXPECT_EQ(pmem.nsegments(), 1u) << "wrong number of segments"; + + ZPhysicalMemory split0_pmem = pmem.split(SegmentSize); + EXPECT_EQ(split0_pmem.nsegments(), 1u); + EXPECT_EQ( pmem.nsegments(), 1u); + EXPECT_EQ(split0_pmem.size(), 1 * SegmentSize); + EXPECT_EQ( pmem.size(), 7 * SegmentSize); + + ZPhysicalMemory split1_pmem = pmem.split(2 * SegmentSize); + EXPECT_EQ(split1_pmem.nsegments(), 1u); + EXPECT_EQ( pmem.nsegments(), 1u); + EXPECT_EQ(split1_pmem.size(), 2 * SegmentSize); + EXPECT_EQ( pmem.size(), 5 * SegmentSize); + + ZPhysicalMemory split2_pmem = pmem.split(5 * SegmentSize); + EXPECT_EQ(split2_pmem.nsegments(), 1u); + EXPECT_EQ( pmem.nsegments(), 1u); + EXPECT_EQ(split2_pmem.size(), 5 * SegmentSize); + EXPECT_EQ( pmem.size(), 0 * SegmentSize); +} + +#endif --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zUtils.cpp 2018-06-01 22:31:26.121845373 +0200 @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "unittest.hpp" + +#include + +template +static T max_alignment() { + T max = std::numeric_limits::max(); + return max ^ (max >> 1); +} + +TEST(ZUtilsTest, round_up_power_of_2) { + EXPECT_EQ(ZUtils::round_up_power_of_2(1u), 1u); + EXPECT_EQ(ZUtils::round_up_power_of_2(2u), 2u); + EXPECT_EQ(ZUtils::round_up_power_of_2(3u), 4u); + EXPECT_EQ(ZUtils::round_up_power_of_2(4u), 4u); + EXPECT_EQ(ZUtils::round_up_power_of_2(5u), 8u); + EXPECT_EQ(ZUtils::round_up_power_of_2(6u), 8u); + EXPECT_EQ(ZUtils::round_up_power_of_2(7u), 8u); + EXPECT_EQ(ZUtils::round_up_power_of_2(8u), 8u); + EXPECT_EQ(ZUtils::round_up_power_of_2(9u), 16u); + EXPECT_EQ(ZUtils::round_up_power_of_2(10u), 16u); + EXPECT_EQ(ZUtils::round_up_power_of_2(1023u), 1024u); + EXPECT_EQ(ZUtils::round_up_power_of_2(1024u), 1024u); + EXPECT_EQ(ZUtils::round_up_power_of_2(1025u), 2048u); + + const size_t max = max_alignment(); + EXPECT_EQ(ZUtils::round_up_power_of_2(max - 1), max); + EXPECT_EQ(ZUtils::round_up_power_of_2(max), max); +} + +TEST(ZUtilsTest, round_down_power_of_2) { + EXPECT_EQ(ZUtils::round_down_power_of_2(1u), 1u); + EXPECT_EQ(ZUtils::round_down_power_of_2(2u), 2u); + EXPECT_EQ(ZUtils::round_down_power_of_2(3u), 2u); + EXPECT_EQ(ZUtils::round_down_power_of_2(4u), 4u); + EXPECT_EQ(ZUtils::round_down_power_of_2(5u), 4u); + EXPECT_EQ(ZUtils::round_down_power_of_2(6u), 4u); + EXPECT_EQ(ZUtils::round_down_power_of_2(7u), 4u); + EXPECT_EQ(ZUtils::round_down_power_of_2(8u), 8u); + EXPECT_EQ(ZUtils::round_down_power_of_2(9u), 8u); + EXPECT_EQ(ZUtils::round_down_power_of_2(10u), 8u); + EXPECT_EQ(ZUtils::round_down_power_of_2(1023u), 512u); + EXPECT_EQ(ZUtils::round_down_power_of_2(1024u), 1024u); + EXPECT_EQ(ZUtils::round_down_power_of_2(1025u), 1024u); + + const size_t max = max_alignment(); + EXPECT_EQ(ZUtils::round_down_power_of_2(max), max); + EXPECT_EQ(ZUtils::round_down_power_of_2(max - 1), max / 2); +} --- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/test/hotspot/gtest/gc/z/test_zVirtualMemory.cpp 2018-06-01 22:31:26.500861733 +0200 @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +#include "precompiled.hpp" +#include "gc/z/zVirtualMemory.inline.hpp" +#include "utilities/debug.hpp" +#include "unittest.hpp" + +TEST(ZVirtualMemory, split) { + const size_t PageSize = 2 * M; + + ZVirtualMemory mem(0, 10 * PageSize); + + ZVirtualMemory mem_split0 = mem.split(0 * PageSize); + EXPECT_EQ(mem_split0.size(), 0 * PageSize); + EXPECT_EQ( mem.size(), 10 * PageSize); + + ZVirtualMemory mem_split1 = mem.split(5u * PageSize); + EXPECT_EQ(mem_split1.size(), 5 * PageSize); + EXPECT_EQ( mem.size(), 5 * PageSize); + + ZVirtualMemory mem_split2 = mem.split(5u * PageSize); + EXPECT_EQ(mem_split2.size(), 5 * PageSize); + EXPECT_EQ( mem.size(), 0 * PageSize); + + ZVirtualMemory mem_split3 = mem.split(0 * PageSize); + EXPECT_EQ(mem_split3.size(), 0 * PageSize); +}