--- old/make/autoconf/hotspot.m4 2019-06-10 14:25:38.522230671 +0100 +++ new/make/autoconf/hotspot.m4 2019-06-10 14:25:38.242227831 +0100 @@ -350,7 +350,8 @@ # Only enable ZGC on supported platforms AC_MSG_CHECKING([if zgc can be built]) - if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then + if (test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64") || \ + (test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xaarch64"); then AC_MSG_RESULT([yes]) else DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES zgc" --- old/src/hotspot/cpu/aarch64/assembler_aarch64.cpp 2019-06-10 14:25:39.274238301 +0100 +++ new/src/hotspot/cpu/aarch64/assembler_aarch64.cpp 2019-06-10 14:25:39.026235784 +0100 @@ -1265,6 +1265,13 @@ __ movptr(r, (uint64_t)target()); break; } + case post: { + // Post-indexed, just copy the contents of the register. Offset added afterwards. + if (_base == r) // it's a nop + break; + __ mov(r, _base); + break; + } default: ShouldNotReachHere(); } --- old/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp 2019-06-10 14:25:40.018245848 +0100 +++ new/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp 2019-06-10 14:25:39.762243251 +0100 @@ -1015,7 +1015,11 @@ if (UseCompressedOops && !wide) { __ decode_heap_oop(dest->as_register()); } - __ verify_oop(dest->as_register()); + + if (!UseZGC) { + // Load barrier has not yet been applied, so ZGC can't verify the oop here + __ verify_oop(dest->as_register()); + } } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { if (UseCompressedClassPointers) { __ decode_klass_not_null(dest->as_register()); @@ -2869,7 +2873,11 @@ void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { - assert(patch_code == lir_patch_none, "Patch code not supported"); + if (patch_code != lir_patch_none) { + deoptimize_trap(info); + return; + } + __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr())); } --- old/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp 2019-06-10 14:25:40.814253923 +0100 +++ new/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp 2019-06-10 14:25:40.546251205 +0100 @@ -46,6 +46,9 @@ #ifdef COMPILER2 #include "opto/runtime.hpp" #endif +#if INCLUDE_ZGC +#include "gc/z/zThreadLocalData.hpp" +#endif #ifdef BUILTIN_SIM #include "../../../../../../simulator/simulator.hpp" @@ -580,6 +583,16 @@ // make sure object is 'reasonable' __ cbz(r0, exit); // if obj is NULL it is OK +#if INCLUDE_ZGC + if (UseZGC) { + // Check if mask is good. + // verifies that ZAddressBadMask & r0 == 0 + __ ldr(c_rarg3, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(c_rarg2, r0, c_rarg3); + __ cbnz(c_rarg2, error); + } +#endif + // Check if the oop is in the right area of memory __ mov(c_rarg3, (intptr_t) Universe::verify_oop_mask()); __ andr(c_rarg2, r0, c_rarg3); --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp 2019-06-10 14:25:41.318259036 +0100 @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "code/codeBlob.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" +#include "gc/z/zBarrierSetRuntime.hpp" +#include "memory/resourceArea.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/z/c1/zBarrierSetC1.hpp" +#endif // COMPILER1 + +#include "gc/z/zThreadLocalData.hpp" + +ZBarrierSetAssembler::ZBarrierSetAssembler() : + _load_barrier_slow_stub(), + _load_barrier_weak_slow_stub() {} + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) __ block_comment(str) +#endif + +#undef __ +#define __ masm-> + +void ZBarrierSetAssembler::load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register dst, + Address src, + Register tmp1, + Register tmp_thread) { + if (!ZBarrierSet::barrier_needed(decorators, type)) { + // Barrier not needed + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + return; + } + + // rscratch1 can be passed as src or dst, so don't use it. + RegSet savedRegs = RegSet::of(rscratch2, rheapbase); + + Label done; + assert_different_registers(rheapbase, rscratch2, dst); + assert_different_registers(rheapbase, rscratch2, src.base()); + + __ push(savedRegs, sp); + + // Load bad mask into scratch register. + __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); + __ lea(rscratch2, src); + __ ldr(dst, src); + + // Test reference against bad mask. If mask bad, then we need to fix it up. + __ tst(dst, rheapbase); + __ br(Assembler::EQ, done); + + __ enter(); + + __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp); + + // call_VM_leaf uses rscratch1. + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, rscratch2); + + // Make sure dst has the return value. + if (dst != r0) { + __ mov(dst, r0); + } + + __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp); + __ leave(); + + + __ bind(done); + + // Restore tmps + __ pop(savedRegs, sp); +} + +#ifdef ASSERT + +void ZBarrierSetAssembler::store_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Address dst, + Register val, + Register tmp1, + Register tmp2) { + // Verify value + if (type == T_OBJECT || type == T_ARRAY) { + // Note that src could be noreg, which means we + // are storing null and can skip verification. + if (val != noreg) { + Label done; + + // tmp1 and tmp2 are often set to noreg. + RegSet savedRegs = RegSet::of(rscratch1); + __ push(savedRegs, sp); + + __ ldr(tmp1, address_bad_mask_from_thread(rthread)); + __ tst(val, tmp1); + __ br(Assembler::EQ, done); + __ stop("Verify oop store failed"); + __ should_not_reach_here(); + __ bind(done); + __ pop(savedRegs, sp); + } + } + + // Store value + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); +} + +#endif // ASSERT + +void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, + DecoratorSet decorators, + bool is_oop, + Register src, + Register dst, + Register count, + RegSet saved_regs) { + if (!is_oop) { + // Barrier not needed + return; + } + + BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {"); + + assert_different_registers(src, count, rscratch1); + + __ pusha(); + + if (count == c_rarg0) { + if (src == c_rarg1) { + // exactly backwards!! + __ mov(rscratch1, c_rarg0); + __ mov(c_rarg0, c_rarg1); + __ mov(c_rarg1, rscratch1); + } else { + __ mov(c_rarg1, count); + __ mov(c_rarg0, src); + } + } else { + __ mov(c_rarg0, src); + __ mov(c_rarg1, count); + } + + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2); + + __ popa(); + BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue"); +} + +void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, + Register jni_env, + Register robj, + Register tmp, + Label& slowpath) { + BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {"); + + assert_different_registers(jni_env, robj, tmp); + + // Resolve jobject + BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath); + + // The Address offset is too large to direct load - -784. Our range is +127, -128. + __ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) - + in_bytes(JavaThread::jni_environment_offset()))); + // Load address bad mask + __ add(tmp, jni_env, tmp); + __ ldr(tmp, Address(tmp)); + + // Check address bad mask + __ tst(robj, tmp); + __ br(Assembler::NE, slowpath); + + BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); +} + +#ifdef COMPILER1 + +#undef __ +#define __ ce->masm()-> + +void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce, + LIR_Opr ref) const { + assert_different_registers(rheapbase, rthread, ref->as_register()); + + __ ldr(rheapbase, address_bad_mask_from_thread(rthread)); + __ tst(ref->as_register(), rheapbase); +} + +void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, + ZLoadBarrierStubC1* stub) const { + // Stub entry + __ bind(*stub->entry()); + + Register ref = stub->ref()->as_register(); + Register ref_addr = noreg; + Register tmp = noreg; + + if (stub->tmp()->is_valid()) { + // Load address into tmp register + ce->leal(stub->ref_addr(), stub->tmp()); + ref_addr = tmp = stub->tmp()->as_pointer_register(); + } else { + // Address already in register + ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); + } + + assert_different_registers(ref, ref_addr, noreg); + + // Save r0 unless it is the result or tmp register + // Set up SP to accomodate parameters and maybe r0.. + if (ref != r0 && tmp != r0) { + __ sub(sp, sp, 32); + __ str(r0, Address(sp, 16)); + } else { + __ sub(sp, sp, 16); + } + + // Setup arguments and call runtime stub + ce->store_parameter(ref_addr, 1); + ce->store_parameter(ref, 0); + + __ far_call(stub->runtime_stub()); + + // Verify result + __ verify_oop(r0, "Bad oop"); + + // Move result into place + if (ref != r0) { + __ mov(ref, r0); + } + + // Restore r0 unless it is the result or tmp register + if (ref != r0 && tmp != r0) { + __ ldr(r0, Address(sp, 16)); + __ add(sp, sp, 32); + } else { + __ add(sp, sp, 16); + } + + // Stub exit + __ b(*stub->continuation()); +} + +#undef __ +#define __ sasm-> + +void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, + DecoratorSet decorators) const { + __ prologue("zgc_load_barrier stub", false); + + // We don't use push/pop_clobbered_registers() - we need to pull out the result from r0. + for (int i = 0; i < 32; i +=2) { + __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16))); + } + + RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0); + __ push(saveRegs, sp); + + // Setup arguments + __ load_parameter(0, c_rarg0); + __ load_parameter(1, c_rarg1); + + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); + + __ pop(saveRegs, sp); + + for (int i = 30; i >0; i -=2) { + __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16))); + } + + __ epilogue(); +} +#endif // COMPILER1 + +#undef __ +#define __ cgen->assembler()-> + +// Generates a register specific stub for calling +// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +// +// The raddr register serves as both input and output for this stub. When the stub is +// called the raddr register contains the object field address (oop*) where the bad oop +// was loaded from, which caused the slow path to be taken. On return from the stub the +// raddr register contains the good/healed oop returned from +// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or +// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). +static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { + // Don't generate stub for invalid registers + if (raddr == zr || raddr == r29 || raddr == r30) { + return NULL; + } + + // Create stub name + char name[64]; + const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; + os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); + + __ align(CodeEntryAlignment); + StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); + address start = __ pc(); + + // Save live registers + RegSet savedRegs = RegSet::range(r0,r28) - RegSet::of(raddr); + + __ enter(); + __ push(savedRegs, sp); + + // Setup arguments + if (raddr != c_rarg1) { + __ mov(c_rarg1, raddr); + } + + __ ldr(c_rarg0, Address(raddr)); + + // Call barrier function + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); + + // Move result returned in r0 to raddr, if needed + if (raddr != r0) { + __ mov(raddr, r0); + } + + __ pop(savedRegs, sp); + __ leave(); + __ ret(lr); + + return start; +} + +#undef __ + +static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) { + const int nregs = 28; // Exclude FP, XZR, SP from calculation. + const int code_size = nregs * 254; // Rough estimate of code size + + ResourceMark rm; + + CodeBuffer buf(BufferBlob::create(label, code_size)); + StubCodeGenerator cgen(&buf); + + for (int i = 0; i < nregs; i++) { + const Register reg = as_Register(i); + stub[i] = generate_load_barrier_stub(&cgen, reg, decorators); + } +} + +void ZBarrierSetAssembler::barrier_stubs_init() { + barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub); + barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub); +} + +address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) { + return _load_barrier_slow_stub[reg->encoding()]; +} + +address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) { + return _load_barrier_weak_slow_stub[reg->encoding()]; +} --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp 2019-06-10 14:25:42.070266665 +0100 @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP +#define CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP + +#ifdef COMPILER1 +class LIR_Assembler; +class LIR_OprDesc; +typedef LIR_OprDesc* LIR_Opr; +class StubAssembler; +class ZLoadBarrierStubC1; +#endif // COMPILER1 + +class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { +private: + address _load_barrier_slow_stub[RegisterImpl::number_of_registers]; + address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers]; + +public: + ZBarrierSetAssembler(); + + virtual void load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register dst, + Address src, + Register tmp1, + Register tmp_thread); + +#ifdef ASSERT + virtual void store_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Address dst, + Register val, + Register tmp1, + Register tmp2); +#endif // ASSERT + + virtual void arraycopy_prologue(MacroAssembler* masm, + DecoratorSet decorators, + bool is_oop, + Register src, + Register dst, + Register count, + RegSet saved_regs); + + virtual void try_resolve_jobject_in_native(MacroAssembler* masm, + Register jni_env, + Register robj, + Register tmp, + Label& slowpath); + +#ifdef COMPILER1 + void generate_c1_load_barrier_test(LIR_Assembler* ce, + LIR_Opr ref) const; + + void generate_c1_load_barrier_stub(LIR_Assembler* ce, + ZLoadBarrierStubC1* stub) const; + + void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, + DecoratorSet decorators) const; +#endif // COMPILER1 + + virtual void barrier_stubs_init(); + + address load_barrier_slow_stub(Register reg); + address load_barrier_weak_slow_stub(Register reg); +}; + +#endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad 2019-06-10 14:25:42.850274577 +0100 @@ -0,0 +1,150 @@ +// +// Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// + +source_hpp %{ + +#include "gc/z/c2/zBarrierSetC2.hpp" + +%} + +source %{ + +#include "gc/z/zBarrierSetAssembler.hpp" + +static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst, + Register base, int index, int scale, + int disp, bool weak) { + const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst) + : ZBarrierSet::assembler()->load_barrier_slow_stub(dst); + + if (index == -1) { + if (disp != 0) { + __ lea(dst, Address(base, disp)); + } else { + __ mov(dst, base); + } + } else { + Register index_reg = as_Register(index); + if (disp == 0) { + __ lea(dst, Address(base, index_reg, Address::lsl(scale))); + } else { + __ lea(dst, Address(base, disp)); + __ lea(dst, Address(dst, index_reg, Address::lsl(scale))); + } + } + + __ far_call(RuntimeAddress(stub)); +} + +%} + +// +// Execute ZGC load barrier (strong) slow path +// +instruct loadBarrierSlowReg(iRegP dst, memory mem, rFlagsReg cr) %{ + match(Set dst (LoadBarrierSlowReg mem)); + predicate(!n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr); + + format %{"LoadBarrierSlowReg $dst, $mem" %} + ins_encode %{ + z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, + $mem$$index, $mem$$scale, $mem$$disp, false); + %} + ins_pipe(pipe_slow); +%} + +// +// Execute ZGC load barrier (weak) slow path +// +instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr) %{ + match(Set dst (LoadBarrierSlowReg mem)); + predicate(n->as_LoadBarrierSlowReg()->is_weak()); + + effect(DEF dst, KILL cr); + + format %{"LoadBarrierWeakSlowReg $dst, $mem" %} + ins_encode %{ + z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register, + $mem$$index, $mem$$scale, $mem$$disp, true); + %} + ins_pipe(pipe_slow); +%} + + +// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed +// but doesn't affect output. + +instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem, + iRegP oldval, iRegP newval, iRegP keepalive, + rFlagsReg cr) %{ + match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval))); + ins_cost(2 * VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct z_compareAndSwapP(iRegINoSp res, + indirect mem, + iRegP oldval, iRegP newval, iRegP keepalive, + rFlagsReg cr) %{ + + match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); + match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); + + ins_cost(2 * VOLATILE_REF_COST); + + effect(KILL cr); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" + "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + + ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), + aarch64_enc_cset_eq(res)); + + ins_pipe(pipe_slow); +%} + + +instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev, + iRegP keepalive) %{ + match(Set prev (ZGetAndSetP mem (Binary newv keepalive))); + + ins_cost(2 * VOLATILE_REF_COST); + format %{ "atomic_xchg $prev, $newv, [$mem]" %} + ins_encode %{ + __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zArguments_linux_aarch64.cpp 2019-06-10 14:25:43.614282327 +0100 @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zArguments.hpp" +#include "runtime/globals.hpp" +#include "runtime/globals_extension.hpp" +#include "utilities/debug.hpp" + +void ZArguments::initialize_platform() { + // Disable class unloading - we don't support concurrent class unloading yet. + FLAG_SET_DEFAULT(ClassUnloading, false); + FLAG_SET_DEFAULT(ClassUnloadingWithConcurrentMark, false); +} --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.cpp 2019-06-10 14:25:44.374290036 +0100 @@ -0,0 +1,590 @@ +/* + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zArray.inline.hpp" +#include "gc/z/zBackingFile_linux_aarch64.hpp" +#include "gc/z/zBackingPath_linux_aarch64.hpp" +#include "gc/z/zErrno.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zLargePages.inline.hpp" +#include "logging/log.hpp" +#include "runtime/init.hpp" +#include "runtime/os.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +#include +#include +#include +#include +#include +#include +#include + +// +// Support for building on older Linux systems +// + +// System calls +#ifndef SYS_fallocate +#define SYS_fallocate 47 +#endif +#ifndef SYS_memfd_create +#define SYS_memfd_create 279 +#endif + +// memfd_create(2) flags +#ifndef MFD_CLOEXEC +#define MFD_CLOEXEC 0x0001U +#endif +#ifndef MFD_HUGETLB +#define MFD_HUGETLB 0x0004U +#endif + +// open(2) flags +#ifndef O_CLOEXEC +#define O_CLOEXEC 02000000 +#endif +#ifndef O_TMPFILE +#define O_TMPFILE (020000000 | O_DIRECTORY) +#endif + +// fallocate(2) flags +#ifndef FALLOC_FL_KEEP_SIZE +#define FALLOC_FL_KEEP_SIZE 0x01 +#endif +#ifndef FALLOC_FL_PUNCH_HOLE +#define FALLOC_FL_PUNCH_HOLE 0x02 +#endif + +// Filesystem types, see statfs(2) +#ifndef TMPFS_MAGIC +#define TMPFS_MAGIC 0x01021994 +#endif +#ifndef HUGETLBFS_MAGIC +#define HUGETLBFS_MAGIC 0x958458f6 +#endif + +// Filesystem names +#define ZFILESYSTEM_TMPFS "tmpfs" +#define ZFILESYSTEM_HUGETLBFS "hugetlbfs" + +// Sysfs file for transparent huge page on tmpfs +#define ZFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled" + +// Java heap filename +#define ZFILENAME_HEAP "java_heap" + +// Preferred tmpfs mount points, ordered by priority +static const char* z_preferred_tmpfs_mountpoints[] = { + "/dev/shm", + "/run/shm", + NULL +}; + +// Preferred hugetlbfs mount points, ordered by priority +static const char* z_preferred_hugetlbfs_mountpoints[] = { + "/dev/hugepages", + "/hugepages", + NULL +}; + +static int z_fallocate_hugetlbfs_attempts = 3; +static bool z_fallocate_supported = true; + +static int z_fallocate(int fd, int mode, size_t offset, size_t length) { + return syscall(SYS_fallocate, fd, mode, offset, length); +} + +static int z_memfd_create(const char *name, unsigned int flags) { + return syscall(SYS_memfd_create, name, flags); +} + +ZBackingFile::ZBackingFile() : + _fd(-1), + _size(0), + _filesystem(0), + _block_size(0), + _available(0), + _initialized(false) { + + // Create backing file + _fd = create_fd(ZFILENAME_HEAP); + if (_fd == -1) { + return; + } + + // Get filesystem statistics + struct statfs buf; + if (fstatfs(_fd, &buf) == -1) { + ZErrno err; + log_error(gc)("Failed to determine filesystem type for backing file (%s)", err.to_string()); + return; + } + + _filesystem = buf.f_type; + _block_size = buf.f_bsize; + _available = buf.f_bavail * _block_size; + + // Make sure we're on a supported filesystem + if (!is_tmpfs() && !is_hugetlbfs()) { + log_error(gc)("Backing file must be located on a %s or a %s filesystem", + ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS); + return; + } + + // Make sure the filesystem type matches requested large page type + if (ZLargePages::is_transparent() && !is_tmpfs()) { + log_error(gc)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem", + ZFILESYSTEM_TMPFS); + return; + } + + if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) { + log_error(gc)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", + ZFILESYSTEM_TMPFS); + return; + } + + if (ZLargePages::is_explicit() && !is_hugetlbfs()) { + log_error(gc)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled " + "when using a %s filesystem", ZFILESYSTEM_HUGETLBFS); + return; + } + + if (!ZLargePages::is_explicit() && is_hugetlbfs()) { + log_error(gc)("-XX:+UseLargePages must be enabled when using a %s filesystem", + ZFILESYSTEM_HUGETLBFS); + return; + } + + const size_t expected_block_size = is_tmpfs() ? os::vm_page_size() : os::large_page_size(); + if (expected_block_size != _block_size) { + log_error(gc)("%s filesystem has unexpected block size " SIZE_FORMAT " (expected " SIZE_FORMAT ")", + is_tmpfs() ? ZFILESYSTEM_TMPFS : ZFILESYSTEM_HUGETLBFS, _block_size, expected_block_size); + return; + } + + // Successfully initialized + _initialized = true; +} + +int ZBackingFile::create_mem_fd(const char* name) const { + // Create file name + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : ""); + + // Create file + const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0; + const int fd = z_memfd_create(filename, MFD_CLOEXEC | extra_flags); + if (fd == -1) { + ZErrno err; + log_debug(gc, init)("Failed to create memfd file (%s)", + ((ZLargePages::is_explicit() && err == EINVAL) ? "Hugepages not supported" : err.to_string())); + return -1; + } + + log_info(gc, init)("Heap backed by file: /memfd:%s", filename); + + return fd; +} + +int ZBackingFile::create_file_fd(const char* name) const { + const char* const filesystem = ZLargePages::is_explicit() + ? ZFILESYSTEM_HUGETLBFS + : ZFILESYSTEM_TMPFS; + const char** const preferred_mountpoints = ZLargePages::is_explicit() + ? z_preferred_hugetlbfs_mountpoints + : z_preferred_tmpfs_mountpoints; + + // Find mountpoint + ZBackingPath path(filesystem, preferred_mountpoints); + if (path.get() == NULL) { + log_error(gc)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem); + return -1; + } + + // Try to create an anonymous file using the O_TMPFILE flag. Note that this + // flag requires kernel >= 3.11. If this fails we fall back to open/unlink. + const int fd_anon = os::open(path.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); + if (fd_anon == -1) { + ZErrno err; + log_debug(gc, init)("Failed to create anonymous file in %s (%s)", path.get(), + (err == EINVAL ? "Not supported" : err.to_string())); + } else { + // Get inode number for anonymous file + struct stat stat_buf; + if (fstat(fd_anon, &stat_buf) == -1) { + ZErrno err; + log_error(gc)("Failed to determine inode number for anonymous file (%s)", err.to_string()); + return -1; + } + + log_info(gc, init)("Heap backed by file: %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino); + + return fd_anon; + } + + log_debug(gc, init)("Falling back to open/unlink"); + + // Create file name + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s/%s.%d", path.get(), name, os::current_process_id()); + + // Create file + const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR); + if (fd == -1) { + ZErrno err; + log_error(gc)("Failed to create file %s (%s)", filename, err.to_string()); + return -1; + } + + // Unlink file + if (unlink(filename) == -1) { + ZErrno err; + log_error(gc)("Failed to unlink file %s (%s)", filename, err.to_string()); + return -1; + } + + log_info(gc, init)("Heap backed by file: %s", filename); + + return fd; +} + +int ZBackingFile::create_fd(const char* name) const { + if (ZPath == NULL) { + // If the path is not explicitly specified, then we first try to create a memfd file + // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might + // not be supported at all (requires kernel >= 3.17), or it might not support large + // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a + // file on an accessible tmpfs or hugetlbfs mount point. + const int fd = create_mem_fd(name); + if (fd != -1) { + return fd; + } + + log_debug(gc, init)("Falling back to searching for an accessible mount point"); + } + + return create_file_fd(name); +} + +bool ZBackingFile::is_initialized() const { + return _initialized; +} + +int ZBackingFile::fd() const { + return _fd; +} + +size_t ZBackingFile::size() const { + return _size; +} + +size_t ZBackingFile::available() const { + return _available; +} + +bool ZBackingFile::is_tmpfs() const { + return _filesystem == TMPFS_MAGIC; +} + +bool ZBackingFile::is_hugetlbfs() const { + return _filesystem == HUGETLBFS_MAGIC; +} + +bool ZBackingFile::tmpfs_supports_transparent_huge_pages() const { + // If the shmem_enabled file exists and is readable then we + // know the kernel supports transparent huge pages for tmpfs. + return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0; +} + +ZErrno ZBackingFile::fallocate_compat_ftruncate(size_t size) const { + while (ftruncate(_fd, size) == -1) { + if (errno != EINTR) { + // Failed + return errno; + } + } + + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate_compat_mmap(size_t offset, size_t length, bool touch) const { + // On hugetlbfs, mapping a file segment will fail immediately, without + // the need to touch the mapped pages first, if there aren't enough huge + // pages available to back the mapping. + void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset); + if (addr == MAP_FAILED) { + // Failed + return errno; + } + + // Once mapped, the huge pages are only reserved. We need to touch them + // to associate them with the file segment. Note that we can not punch + // hole in file segments which only have reserved pages. + if (touch) { + char* const start = (char*)addr; + char* const end = start + length; + os::pretouch_memory(start, end, _block_size); + } + + // Unmap again. From now on, the huge pages that were mapped are allocated + // to this file. There's no risk in getting SIGBUS when touching them. + if (munmap(addr, length) == -1) { + // Failed + return errno; + } + + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate_compat_pwrite(size_t offset, size_t length) const { + uint8_t data = 0; + + // Allocate backing memory by writing to each block + for (size_t pos = offset; pos < offset + length; pos += _block_size) { + if (pwrite(_fd, &data, sizeof(data), pos) == -1) { + // Failed + return errno; + } + } + + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate_fill_hole_compat(size_t offset, size_t length) { + // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs + // since Linux 4.3. When fallocate(2) is not supported we emulate it using + // ftruncate/pwrite (for tmpfs) or ftruncate/mmap/munmap (for hugetlbfs). + + const size_t end = offset + length; + if (end > _size) { + // Increase file size + const ZErrno err = fallocate_compat_ftruncate(end); + if (err) { + // Failed + return err; + } + } + + // Allocate backing memory + const ZErrno err = is_hugetlbfs() ? fallocate_compat_mmap(offset, length, false /* touch */) + : fallocate_compat_pwrite(offset, length); + if (err) { + if (end > _size) { + // Restore file size + fallocate_compat_ftruncate(_size); + } + + // Failed + return err; + } + + if (end > _size) { + // Record new file size + _size = end; + } + + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate_fill_hole_syscall(size_t offset, size_t length) { + const int mode = 0; // Allocate + const int res = z_fallocate(_fd, mode, offset, length); + if (res == -1) { + // Failed + return errno; + } + + const size_t end = offset + length; + if (end > _size) { + // Record new file size + _size = end; + } + + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate_fill_hole(size_t offset, size_t length) { + // Using compat mode is more efficient when allocating space on hugetlbfs. + // Note that allocating huge pages this way will only reserve them, and not + // associate them with segments of the file. We must guarantee that we at + // some point touch these segments, otherwise we can not punch hole in them. + if (z_fallocate_supported && !is_hugetlbfs()) { + const ZErrno err = fallocate_fill_hole_syscall(offset, length); + if (!err) { + // Success + return 0; + } + + if (err != ENOSYS && err != EOPNOTSUPP) { + // Failed + return err; + } + + // Not supported + log_debug(gc)("Falling back to fallocate() compatibility mode"); + z_fallocate_supported = false; + } + + return fallocate_fill_hole_compat(offset, length); +} + +ZErrno ZBackingFile::fallocate_punch_hole(size_t offset, size_t length) { + if (is_hugetlbfs()) { + // We can only punch hole in pages that have been touched. Non-touched + // pages are only reserved, and not associated with any specific file + // segment. We don't know which pages have been previously touched, so + // we always touch them here to guarantee that we can punch hole. + const ZErrno err = fallocate_compat_mmap(offset, length, true /* touch */); + if (err) { + // Failed + return err; + } + } + + const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE; + if (z_fallocate(_fd, mode, offset, length) == -1) { + // Failed + return errno; + } + + // Success + return 0; +} + +ZErrno ZBackingFile::split_and_fallocate(bool punch_hole, size_t offset, size_t length) { + // Try first half + const size_t offset0 = offset; + const size_t length0 = align_up(length / 2, _block_size); + const ZErrno err0 = fallocate(punch_hole, offset0, length0); + if (err0) { + return err0; + } + + // Try second half + const size_t offset1 = offset0 + length0; + const size_t length1 = length - length0; + const ZErrno err1 = fallocate(punch_hole, offset1, length1); + if (err1) { + return err1; + } + + // Success + return 0; +} + +ZErrno ZBackingFile::fallocate(bool punch_hole, size_t offset, size_t length) { + assert(is_aligned(offset, _block_size), "Invalid offset"); + assert(is_aligned(length, _block_size), "Invalid length"); + + const ZErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length); + if (err == EINTR && length > _block_size) { + // Calling fallocate(2) with a large length can take a long time to + // complete. When running profilers, such as VTune, this syscall will + // be constantly interrupted by signals. Expanding the file in smaller + // steps avoids this problem. + return split_and_fallocate(punch_hole, offset, length); + } + + return err; +} + +bool ZBackingFile::commit_inner(size_t offset, size_t length) { + log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", + offset / M, (offset + length) / M, length / M); + +retry: + const ZErrno err = fallocate(false /* punch_hole */, offset, length); + if (err) { + if (err == ENOSPC && !is_init_completed() && is_hugetlbfs() && z_fallocate_hugetlbfs_attempts-- > 0) { + // If we fail to allocate during initialization, due to lack of space on + // the hugetlbfs filesystem, then we wait and retry a few times before + // giving up. Otherwise there is a risk that running JVMs back-to-back + // will fail, since there is a delay between process termination and the + // huge pages owned by that process being returned to the huge page pool + // and made available for new allocations. + log_debug(gc, init)("Failed to commit memory (%s), retrying", err.to_string()); + + // Wait and retry in one second, in the hope that huge pages will be + // available by then. + sleep(1); + goto retry; + } + + // Failed + log_error(gc)("Failed to commit memory (%s)", err.to_string()); + return false; + } + + // Success + return true; +} + +size_t ZBackingFile::commit(size_t offset, size_t length) { + // Try to commit the whole region + if (commit_inner(offset, length)) { + // Success + return length; + } + + // Failed, try to commit as much as possible + size_t start = offset; + size_t end = offset + length; + + for (;;) { + length = align_down((end - start) / 2, ZGranuleSize); + if (length < ZGranuleSize) { + // Done, don't commit more + return start - offset; + } + + if (commit_inner(start, length)) { + // Success, try commit more + start += length; + } else { + // Failed, try commit less + end -= length; + } + } +} + +size_t ZBackingFile::uncommit(size_t offset, size_t length) { + log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)", + offset / M, (offset + length) / M, length / M); + + const ZErrno err = fallocate(true /* punch_hole */, offset, length); + if (err) { + log_error(gc)("Failed to uncommit memory (%s)", err.to_string()); + return 0; + } + + return length; +} --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingFile_linux_aarch64.hpp 2019-06-10 14:25:45.230298717 +0100 @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGFILE_LINUX_AARCH64_HPP +#define OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGFILE_LINUX_AARCH64_HPP + +#include "memory/allocation.hpp" + +class ZErrno; + +class ZBackingFile { +private: + int _fd; + size_t _size; + uint64_t _filesystem; + size_t _block_size; + size_t _available; + bool _initialized; + + int create_mem_fd(const char* name) const; + int create_file_fd(const char* name) const; + int create_fd(const char* name) const; + + bool is_tmpfs() const; + bool is_hugetlbfs() const; + bool tmpfs_supports_transparent_huge_pages() const; + + ZErrno fallocate_compat_ftruncate(size_t size) const; + ZErrno fallocate_compat_mmap(size_t offset, size_t length, bool reserve_only) const; + ZErrno fallocate_compat_pwrite(size_t offset, size_t length) const; + ZErrno fallocate_fill_hole_compat(size_t offset, size_t length); + ZErrno fallocate_fill_hole_syscall(size_t offset, size_t length); + ZErrno fallocate_fill_hole(size_t offset, size_t length); + ZErrno fallocate_punch_hole(size_t offset, size_t length); + ZErrno split_and_fallocate(bool punch_hole, size_t offset, size_t length); + ZErrno fallocate(bool punch_hole, size_t offset, size_t length); + + bool commit_inner(size_t offset, size_t length); + +public: + ZBackingFile(); + + bool is_initialized() const; + + int fd() const; + size_t size() const; + size_t available() const; + + size_t commit(size_t offset, size_t length); + size_t uncommit(size_t offset, size_t length); +}; + +#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGFILE_LINUX_AARCH64_HPP --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.cpp 2019-06-10 14:25:46.010306628 +0100 @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zArray.inline.hpp" +#include "zBackingPath_linux_aarch64.hpp" +#include "gc/z/zErrno.hpp" +#include "logging/log.hpp" + +#include +#include + +// Mount information, see proc(5) for more details. +#define PROC_SELF_MOUNTINFO "/proc/self/mountinfo" + +ZBackingPath::ZBackingPath(const char* filesystem, const char** preferred_mountpoints) { + if (ZPath != NULL) { + // Use specified path + _path = strdup(ZPath); + } else { + // Find suitable path + _path = find_mountpoint(filesystem, preferred_mountpoints); + } +} + +ZBackingPath::~ZBackingPath() { + free(_path); + _path = NULL; +} + +char* ZBackingPath::get_mountpoint(const char* line, const char* filesystem) const { + char* line_mountpoint = NULL; + char* line_filesystem = NULL; + + // Parse line and return a newly allocated string containing the mount point if + // the line contains a matching filesystem and the mount point is accessible by + // the current user. + if (sscanf(line, "%*u %*u %*u:%*u %*s %ms %*[^-]- %ms", &line_mountpoint, &line_filesystem) != 2 || + strcmp(line_filesystem, filesystem) != 0 || + access(line_mountpoint, R_OK|W_OK|X_OK) != 0) { + // Not a matching or accessible filesystem + free(line_mountpoint); + line_mountpoint = NULL; + } + + free(line_filesystem); + + return line_mountpoint; +} + +void ZBackingPath::get_mountpoints(const char* filesystem, ZArray* mountpoints) const { + FILE* fd = fopen(PROC_SELF_MOUNTINFO, "r"); + if (fd == NULL) { + ZErrno err; + log_error(gc)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string()); + return; + } + + char* line = NULL; + size_t length = 0; + + while (getline(&line, &length, fd) != -1) { + char* const mountpoint = get_mountpoint(line, filesystem); + if (mountpoint != NULL) { + mountpoints->add(mountpoint); + } + } + + free(line); + fclose(fd); +} + +void ZBackingPath::free_mountpoints(ZArray* mountpoints) const { + ZArrayIterator iter(mountpoints); + for (char* mountpoint; iter.next(&mountpoint);) { + free(mountpoint); + } + mountpoints->clear(); +} + +char* ZBackingPath::find_preferred_mountpoint(const char* filesystem, + ZArray* mountpoints, + const char** preferred_mountpoints) const { + // Find preferred mount point + ZArrayIterator iter1(mountpoints); + for (char* mountpoint; iter1.next(&mountpoint);) { + for (const char** preferred = preferred_mountpoints; *preferred != NULL; preferred++) { + if (!strcmp(mountpoint, *preferred)) { + // Preferred mount point found + return strdup(mountpoint); + } + } + } + + // Preferred mount point not found + log_error(gc)("More than one %s filesystem found:", filesystem); + ZArrayIterator iter2(mountpoints); + for (char* mountpoint; iter2.next(&mountpoint);) { + log_error(gc)(" %s", mountpoint); + } + + return NULL; +} + +char* ZBackingPath::find_mountpoint(const char* filesystem, const char** preferred_mountpoints) const { + char* path = NULL; + ZArray mountpoints; + + get_mountpoints(filesystem, &mountpoints); + + if (mountpoints.size() == 0) { + // No mount point found + log_error(gc)("Failed to find an accessible %s filesystem", filesystem); + } else if (mountpoints.size() == 1) { + // One mount point found + path = strdup(mountpoints.at(0)); + } else { + // More than one mount point found + path = find_preferred_mountpoint(filesystem, &mountpoints, preferred_mountpoints); + } + + free_mountpoints(&mountpoints); + + return path; +} + +const char* ZBackingPath::get() const { + return _path; +} --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zBackingPath_linux_aarch64.hpp 2019-06-10 14:25:46.754314175 +0100 @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGPATH_LINUX_AARCH64_HPP +#define OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGPATH_LINUX_AARCH64_HPP + +#include "gc/z/zArray.hpp" +#include "memory/allocation.hpp" + +class ZBackingPath : public StackObj { +private: + char* _path; + + char* get_mountpoint(const char* line, + const char* filesystem) const; + void get_mountpoints(const char* filesystem, + ZArray* mountpoints) const; + void free_mountpoints(ZArray* mountpoints) const; + char* find_preferred_mountpoint(const char* filesystem, + ZArray* mountpoints, + const char** preferred_mountpoints) const; + char* find_mountpoint(const char* filesystem, + const char** preferred_mountpoints) const; + +public: + ZBackingPath(const char* filesystem, const char** preferred_mountpoints); + ~ZBackingPath(); + + const char* get() const; +}; + +#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZBACKINGPATH_LINUX_AARCH64_HPP --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.cpp 2019-06-10 14:25:47.574322490 +0100 @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "runtime/globals.hpp" +#include "utilities/globalDefinitions.hpp" + +// +// The heap can have three different layouts, depending on the max heap size. +// +// Address Space & Pointer Layout 1 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000014000000000 (20TB) +// | Remapped View | +// +--------------------------------+ 0x0000010000000000 (16TB) +// | (Reserved, but unused) | +// +--------------------------------+ 0x00000c0000000000 (12TB) +// | Marked1 View | +// +--------------------------------+ 0x0000080000000000 (8TB) +// | Marked0 View | +// +--------------------------------+ 0x0000040000000000 (4TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 4 4 4 4 +// 3 6 5 2 1 0 +// +--------------------+----+-----------------------------------------------+ +// |00000000 00000000 00|1111|11 11111111 11111111 11111111 11111111 11111111| +// +--------------------+----+-----------------------------------------------+ +// | | | +// | | * 41-0 Object Offset (42-bits, 4TB address space) +// | | +// | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB) +// | 0010 = Marked1 (Address view 8-12TB) +// | 0100 = Remapped (Address view 16-20TB) +// | 1000 = Finalizable (Address view N/A) +// | +// * 63-46 Fixed (18-bits, always zero) +// +// +// Address Space & Pointer Layout 2 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000280000000000 (40TB) +// | Remapped View | +// +--------------------------------+ 0x0000200000000000 (32TB) +// | (Reserved, but unused) | +// +--------------------------------+ 0x0000180000000000 (24TB) +// | Marked1 View | +// +--------------------------------+ 0x0000100000000000 (16TB) +// | Marked0 View | +// +--------------------------------+ 0x0000080000000000 (8TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 4 4 4 4 +// 3 7 6 3 2 0 +// +------------------+-----+------------------------------------------------+ +// |00000000 00000000 0|1111|111 11111111 11111111 11111111 11111111 11111111| +// +-------------------+----+------------------------------------------------+ +// | | | +// | | * 42-0 Object Offset (43-bits, 8TB address space) +// | | +// | * 46-43 Metadata Bits (4-bits) 0001 = Marked0 (Address view 8-16TB) +// | 0010 = Marked1 (Address view 16-24TB) +// | 0100 = Remapped (Address view 32-40TB) +// | 1000 = Finalizable (Address view N/A) +// | +// * 63-47 Fixed (17-bits, always zero) +// +// +// Address Space & Pointer Layout 3 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000500000000000 (80TB) +// | Remapped View | +// +--------------------------------+ 0x0000400000000000 (64TB) +// | (Reserved, but unused) | +// +--------------------------------+ 0x0000300000000000 (48TB) +// | Marked1 View | +// +--------------------------------+ 0x0000200000000000 (32TB) +// | Marked0 View | +// +--------------------------------+ 0x0000100000000000 (16TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 4 4 4 4 +// 3 8 7 4 3 0 +// +------------------+----+-------------------------------------------------+ +// |00000000 00000000 |1111|1111 11111111 11111111 11111111 11111111 11111111| +// +------------------+----+-------------------------------------------------+ +// | | | +// | | * 43-0 Object Offset (44-bits, 16TB address space) +// | | +// | * 47-44 Metadata Bits (4-bits) 0001 = Marked0 (Address view 16-32TB) +// | 0010 = Marked1 (Address view 32-48TB) +// | 0100 = Remapped (Address view 64-80TB) +// | 1000 = Finalizable (Address view N/A) +// | +// * 63-48 Fixed (16-bits, always zero) +// + +uintptr_t ZPlatformAddressSpaceStart() { + const uintptr_t first_heap_view_address = (uintptr_t)1 << (ZPlatformAddressMetadataShift() + 0); + const size_t min_address_offset = 0; + return first_heap_view_address + min_address_offset; +} + +uintptr_t ZPlatformAddressSpaceEnd() { + const uintptr_t last_heap_view_address = (uintptr_t)1 << (ZPlatformAddressMetadataShift() + 2); + const size_t max_address_offset = (size_t)1 << ZPlatformAddressOffsetBits(); + return last_heap_view_address + max_address_offset; +} + +uintptr_t ZPlatformAddressReservedStart() { + return ZPlatformAddressSpaceStart(); +} + +uintptr_t ZPlatformAddressReservedEnd() { + return ZPlatformAddressSpaceEnd(); +} + +uintptr_t ZPlatformAddressBase() { + return 0; +} + +size_t ZPlatformAddressOffsetBits() { + const size_t min_address_offset_bits = 42; // 4TB + const size_t max_address_offset_bits = 44; // 16TB + const size_t virtual_to_physical_ratio = 7; // 7:1 + const size_t address_offset = ZUtils::round_up_power_of_2(MaxHeapSize * virtual_to_physical_ratio); + const size_t address_offset_bits = log2_intptr(address_offset); + return MIN2(MAX2(address_offset_bits, min_address_offset_bits), max_address_offset_bits); +} + +size_t ZPlatformAddressMetadataShift() { + return ZPlatformAddressOffsetBits(); +} --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zGlobals_linux_aarch64.hpp 2019-06-10 14:25:48.386330725 +0100 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZGLOBALS_LINUX_AARCH64_HPP +#define OS_CPU_LINUX_AARCH64_GC_Z_ZGLOBALS_LINUX_AARCH64_HPP + +// +// Page Allocation Tiers +// --------------------- +// +// Page Type Page Size Object Size Limit Object Alignment +// ------------------------------------------------------------------ +// Small 2M <= 265K +// Medium 32M <= 4M 4K +// Large X*M > 4M 2M +// ------------------------------------------------------------------ +// +const size_t ZPlatformGranuleSizeShift = 21; // 2MB +const size_t ZPlatformMaxHeapSizeShift = 46; // 16TB +const size_t ZPlatformNMethodDisarmedOffset = 4; +const size_t ZPlatformCacheLineSize = 64; + +uintptr_t ZPlatformAddressSpaceStart(); +uintptr_t ZPlatformAddressSpaceEnd(); +uintptr_t ZPlatformAddressReservedStart(); +uintptr_t ZPlatformAddressReservedEnd(); +uintptr_t ZPlatformAddressBase(); +size_t ZPlatformAddressOffsetBits(); +size_t ZPlatformAddressMetadataShift(); + +#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZGLOBALS_LINUX_AARCH64_HPP --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.cpp 2019-06-10 14:25:49.210339082 +0100 @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zBackingFile_linux_aarch64.hpp" +#include "gc/z/zErrno.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zLargePages.inline.hpp" +#include "gc/z/zMemory.hpp" +#include "gc/z/zNUMA.hpp" +#include "gc/z/zPhysicalMemory.inline.hpp" +#include "gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp" +#include "logging/log.hpp" +#include "runtime/init.hpp" +#include "runtime/os.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +#include +#include +#include + +// +// Support for building on older Linux systems +// + +// madvise(2) flags +#ifndef MADV_HUGEPAGE +#define MADV_HUGEPAGE 14 +#endif + +// Proc file entry for max map mount +#define ZFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count" + +bool ZPhysicalMemoryBacking::is_initialized() const { + return _file.is_initialized(); +} + +void ZPhysicalMemoryBacking::warn_available_space(size_t max) const { + // Note that the available space on a tmpfs or a hugetlbfs filesystem + // will be zero if no size limit was specified when it was mounted. + const size_t available = _file.available(); + if (available == 0) { + // No size limit set, skip check + log_info(gc, init)("Available space on backing filesystem: N/A"); + return; + } + + log_info(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", available / M); + + // Warn if the filesystem doesn't currently have enough space available to hold + // the max heap size. The max heap size will be capped if we later hit this limit + // when trying to expand the heap. + if (available < max) { + log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); + log_warning(gc)("Not enough space available on the backing filesystem to hold the current max Java heap"); + log_warning(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly " + "(available", max / M); + log_warning(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem " + "size could", available / M); + log_warning(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to map memory."); + } +} + +void ZPhysicalMemoryBacking::warn_max_map_count(size_t max) const { + const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT; + FILE* const file = fopen(filename, "r"); + if (file == NULL) { + // Failed to open file, skip check + log_debug(gc, init)("Failed to open %s", filename); + return; + } + + size_t actual_max_map_count = 0; + const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count); + fclose(file); + if (result != 1) { + // Failed to read file, skip check + log_debug(gc, init)("Failed to read %s", filename); + return; + } + + // The required max map count is impossible to calculate exactly since subsystems + // other than ZGC are also creating memory mappings, and we have no control over that. + // However, ZGC tends to create the most mappings and dominate the total count. + // In the worst cases, ZGC will map each granule three times, i.e. once per heap view. + // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory. + const size_t required_max_map_count = (max / ZGranuleSize) * 3 * 1.2; + if (actual_max_map_count < required_max_map_count) { + log_warning(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****"); + log_warning(gc)("The system limit on number of memory mappings per process might be too low for the given"); + log_warning(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at", + max / M, filename); + log_warning(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution " + "with the current", required_max_map_count, actual_max_map_count); + log_warning(gc)("limit could lead to a fatal error, due to failure to map memory."); + } +} + +void ZPhysicalMemoryBacking::warn_commit_limits(size_t max) const { + // Warn if available space is too low + warn_available_space(max); + + // Warn if max map count is too low + warn_max_map_count(max); +} + +bool ZPhysicalMemoryBacking::supports_uncommit() { + assert(!is_init_completed(), "Invalid state"); + assert(_file.size() >= ZGranuleSize, "Invalid size"); + + // Test if uncommit is supported by uncommitting and then re-committing a granule + return commit(uncommit(ZGranuleSize)) == ZGranuleSize; +} + +size_t ZPhysicalMemoryBacking::commit(size_t size) { + size_t committed = 0; + + // Fill holes in the backing file + while (committed < size) { + size_t allocated = 0; + const size_t remaining = size - committed; + const uintptr_t start = _uncommitted.alloc_from_front_at_most(remaining, &allocated); + if (start == UINTPTR_MAX) { + // No holes to commit + break; + } + + // Try commit hole + const size_t filled = _file.commit(start, allocated); + if (filled > 0) { + // Successful or partialy successful + _committed.free(start, filled); + committed += filled; + } + if (filled < allocated) { + // Failed or partialy failed + _uncommitted.free(start + filled, allocated - filled); + return committed; + } + } + + // Expand backing file + if (committed < size) { + const size_t remaining = size - committed; + const uintptr_t start = _file.size(); + const size_t expanded = _file.commit(start, remaining); + if (expanded > 0) { + // Successful or partialy successful + _committed.free(start, expanded); + committed += expanded; + } + } + + return committed; +} + +size_t ZPhysicalMemoryBacking::uncommit(size_t size) { + size_t uncommitted = 0; + + // Punch holes in backing file + while (uncommitted < size) { + size_t allocated = 0; + const size_t remaining = size - uncommitted; + const uintptr_t start = _committed.alloc_from_back_at_most(remaining, &allocated); + assert(start != UINTPTR_MAX, "Allocation should never fail"); + + // Try punch hole + const size_t punched = _file.uncommit(start, allocated); + if (punched > 0) { + // Successful or partialy successful + _uncommitted.free(start, punched); + uncommitted += punched; + } + if (punched < allocated) { + // Failed or partialy failed + _committed.free(start + punched, allocated - punched); + return uncommitted; + } + } + + return uncommitted; +} + +ZPhysicalMemory ZPhysicalMemoryBacking::alloc(size_t size) { + assert(is_aligned(size, ZGranuleSize), "Invalid size"); + + ZPhysicalMemory pmem; + + // Allocate segments + for (size_t allocated = 0; allocated < size; allocated += ZGranuleSize) { + const uintptr_t start = _committed.alloc_from_front(ZGranuleSize); + assert(start != UINTPTR_MAX, "Allocation should never fail"); + pmem.add_segment(ZPhysicalMemorySegment(start, ZGranuleSize)); + } + + return pmem; +} + +void ZPhysicalMemoryBacking::free(const ZPhysicalMemory& pmem) { + const size_t nsegments = pmem.nsegments(); + + // Free segments + for (size_t i = 0; i < nsegments; i++) { + const ZPhysicalMemorySegment& segment = pmem.segment(i); + _committed.free(segment.start(), segment.size()); + } +} + +void ZPhysicalMemoryBacking::map_failed(ZErrno err) const { + if (err == ENOMEM) { + fatal("Failed to map memory. Please check the system limit on number of " + "memory mappings allowed per process (see %s)", ZFILENAME_PROC_MAX_MAP_COUNT); + } else { + fatal("Failed to map memory (%s)", err.to_string()); + } +} + +void ZPhysicalMemoryBacking::advise_view(uintptr_t addr, size_t size, int advice) const { + if (madvise((void*)addr, size, advice) == -1) { + ZErrno err; + log_error(gc)("Failed to advise on memory (advice %d, %s)", advice, err.to_string()); + } +} + +void ZPhysicalMemoryBacking::pretouch_view(uintptr_t addr, size_t size) const { + const size_t page_size = ZLargePages::is_explicit() ? os::large_page_size() : os::vm_page_size(); + os::pretouch_memory((void*)addr, (void*)(addr + size), page_size); +} + +void ZPhysicalMemoryBacking::map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const { + const size_t nsegments = pmem.nsegments(); + size_t size = 0; + + // Map segments + for (size_t i = 0; i < nsegments; i++) { + const ZPhysicalMemorySegment& segment = pmem.segment(i); + const uintptr_t segment_addr = addr + size; + const void* const res = mmap((void*)segment_addr, segment.size(), PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _file.fd(), segment.start()); + if (res == MAP_FAILED) { + ZErrno err; + map_failed(err); + } + + size += segment.size(); + } + + // Advise on use of transparent huge pages before touching it + if (ZLargePages::is_transparent()) { + advise_view(addr, size, MADV_HUGEPAGE); + } + + // NUMA interleave memory before touching it + ZNUMA::memory_interleave(addr, size); + + // Pre-touch memory + if (pretouch) { + pretouch_view(addr, size); + } +} + +void ZPhysicalMemoryBacking::unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const { + // Note that we must keep the address space reservation intact and just detach + // the backing memory. For this reason we map a new anonymous, non-accessible + // and non-reserved page over the mapping instead of actually unmapping. + const void* const res = mmap((void*)addr, pmem.size(), PROT_NONE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); + if (res == MAP_FAILED) { + ZErrno err; + map_failed(err); + } +} + +uintptr_t ZPhysicalMemoryBacking::nmt_address(uintptr_t offset) const { + // From an NMT point of view we treat the first heap view (marked0) as committed + return ZAddress::marked0(offset); +} + +void ZPhysicalMemoryBacking::map(const ZPhysicalMemory& pmem, uintptr_t offset) const { + if (ZVerifyViews) { + // Map good view + map_view(pmem, ZAddress::good(offset), AlwaysPreTouch); + } else { + // Map all views + map_view(pmem, ZAddress::marked0(offset), AlwaysPreTouch); + map_view(pmem, ZAddress::marked1(offset), AlwaysPreTouch); + map_view(pmem, ZAddress::remapped(offset), AlwaysPreTouch); + } +} + +void ZPhysicalMemoryBacking::unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const { + if (ZVerifyViews) { + // Unmap good view + unmap_view(pmem, ZAddress::good(offset)); + } else { + // Unmap all views + unmap_view(pmem, ZAddress::marked0(offset)); + unmap_view(pmem, ZAddress::marked1(offset)); + unmap_view(pmem, ZAddress::remapped(offset)); + } +} + +void ZPhysicalMemoryBacking::debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const { + // Map good view + assert(ZVerifyViews, "Should be enabled"); + map_view(pmem, ZAddress::good(offset), false /* pretouch */); +} + +void ZPhysicalMemoryBacking::debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const { + // Unmap good view + assert(ZVerifyViews, "Should be enabled"); + unmap_view(pmem, ZAddress::good(offset)); +} --- /dev/null 2019-06-10 08:42:37.317240407 +0100 +++ new/src/hotspot/os_cpu/linux_aarch64/gc/z/zPhysicalMemoryBacking_linux_aarch64.hpp 2019-06-10 14:25:50.010347194 +0100 @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef OS_CPU_LINUX_AARCH64_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP +#define OS_CPU_LINUX_AARCH64_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP + +#include "gc/z/zBackingFile_linux_aarch64.hpp" +#include "gc/z/zMemory.hpp" + +class ZErrno; +class ZPhysicalMemory; + +class ZPhysicalMemoryBacking { +private: + ZBackingFile _file; + ZMemoryManager _committed; + ZMemoryManager _uncommitted; + + void warn_available_space(size_t max) const; + void warn_max_map_count(size_t max) const; + + void map_failed(ZErrno err) const; + + void advise_view(uintptr_t addr, size_t size, int advice) const; + void pretouch_view(uintptr_t addr, size_t size) const; + void map_view(const ZPhysicalMemory& pmem, uintptr_t addr, bool pretouch) const; + void unmap_view(const ZPhysicalMemory& pmem, uintptr_t addr) const; + +public: + bool is_initialized() const; + + void warn_commit_limits(size_t max) const; + bool supports_uncommit(); + + size_t commit(size_t size); + size_t uncommit(size_t size); + + ZPhysicalMemory alloc(size_t size); + void free(const ZPhysicalMemory& pmem); + + uintptr_t nmt_address(uintptr_t offset) const; + + void map(const ZPhysicalMemory& pmem, uintptr_t offset) const; + void unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const; + + void debug_map(const ZPhysicalMemory& pmem, uintptr_t offset) const; + void debug_unmap(const ZPhysicalMemory& pmem, uintptr_t offset) const; +}; + +#endif // OS_CPU_LINUX_AARCH64_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_AARCH64_HPP